<commit_before>// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/utility/local_discovery/service_discovery_message_handler.h"

#include <algorithm>

#include "base/lazy_instance.h"
#include "chrome/common/local_discovery/local_discovery_messages.h"
#include "chrome/utility/local_discovery/service_discovery_client_impl.h"
#include "content/public/utility/utility_thread.h"
#include "net/socket/socket_descriptor.h"

namespace local_discovery {

namespace {

void ClosePlatformSocket(net::SocketDescriptor socket);

class SocketFactory : public net::PlatformSocketFactory {
 public:
  SocketFactory()
      : socket_v4_(net::kInvalidSocket),
        socket_v6_(net::kInvalidSocket) {
  }

  void SetSockets(net::SocketDescriptor socket_v4,
                  net::SocketDescriptor socket_v6) {
    Reset();
    socket_v4_ = socket_v4;
    socket_v6_ = socket_v6;
    VLOG(1) << "SetSockets: " << socket_v4_ << " " << socket_v6_;
  }

  void Reset() {
    if (socket_v4_ != net::kInvalidSocket) {
      ClosePlatformSocket(socket_v4_);
      socket_v4_ = net::kInvalidSocket;
    }
    if (socket_v6_ != net::kInvalidSocket) {
      ClosePlatformSocket(socket_v6_);
      socket_v6_ = net::kInvalidSocket;
    }
  }

  virtual ~SocketFactory() {
    Reset();
  }

 protected:
  virtual net::SocketDescriptor CreateSocket(int family, int type,
                                             int protocol) OVERRIDE {
    net::SocketDescriptor result = net::kInvalidSocket;
    if (type != SOCK_DGRAM) {
      NOTREACHED();
    } else if (family == AF_INET) {
      std::swap(result, socket_v4_);
    } else if (family == AF_INET6) {
      std::swap(result, socket_v6_);
    }
    return result;
  }

 private:
  net::SocketDescriptor socket_v4_;
  net::SocketDescriptor socket_v6_;

  DISALLOW_COPY_AND_ASSIGN(SocketFactory);
};

base::LazyInstance<SocketFactory>
    g_local_discovery_socket_factory = LAZY_INSTANCE_INITIALIZER;

class ScopedSocketFactorySetter {
 public:
  ScopedSocketFactorySetter() {
    net::PlatformSocketFactory::SetInstance(
        &g_local_discovery_socket_factory.Get());
  }

  ~ScopedSocketFactorySetter() {
    net::PlatformSocketFactory::SetInstance(NULL);
    g_local_discovery_socket_factory.Get().Reset();
  }

 private:
  DISALLOW_COPY_AND_ASSIGN(ScopedSocketFactorySetter);
};

#if defined(OS_WIN)

void ClosePlatformSocket(net::SocketDescriptor socket) {
  ::closesocket(socket);
}

void StaticInitializeSocketFactory() {
  g_local_discovery_socket_factory.Get().SetSockets(
      net::CreatePlatformSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP),
      net::CreatePlatformSocket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP));
}

#else  // OS_WIN

void ClosePlatformSocket(net::SocketDescriptor socket) {
  ::close(socket);
}

void StaticInitializeSocketFactory() {
}

#endif  // OS_WIN

void SendHostMessageOnUtilityThread(IPC::Message* msg) {
  content::UtilityThread::Get()->Send(msg);
}

std::string WatcherUpdateToString(ServiceWatcher::UpdateType update) {
  switch (update) {
    case ServiceWatcher::UPDATE_ADDED:
      return "UPDATE_ADDED";
    case ServiceWatcher::UPDATE_CHANGED:
      return "UPDATE_CHANGED";
    case ServiceWatcher::UPDATE_REMOVED:
      return "UPDATE_REMOVED";
    case ServiceWatcher::UPDATE_INVALIDATED:
      return "UPDATE_INVALIDATED";
  }
  return "Unknown Update";
}

std::string ResolverStatusToString(ServiceResolver::RequestStatus status) {
  switch (status) {
    case ServiceResolver::STATUS_SUCCESS:
      return "STATUS_SUCESS";
    case ServiceResolver::STATUS_REQUEST_TIMEOUT:
      return "STATUS_REQUEST_TIMEOUT";
    case ServiceResolver::STATUS_KNOWN_NONEXISTENT:
      return "STATUS_KNOWN_NONEXISTENT";
  }
  return "Unknown Status";
}

}  // namespace

ServiceDiscoveryMessageHandler::ServiceDiscoveryMessageHandler() {
}

ServiceDiscoveryMessageHandler::~ServiceDiscoveryMessageHandler() {
  DCHECK(!discovery_thread_);
}

void ServiceDiscoveryMessageHandler::PreSandboxStartup() {
  StaticInitializeSocketFactory();
}

void ServiceDiscoveryMessageHandler::InitializeMdns() {
  if (service_discovery_client_ || mdns_client_)
    return;

  mdns_client_ = net::MDnsClient::CreateDefault();
  {
    // Temporarily redirect network code to use pre-created sockets.
    ScopedSocketFactorySetter socket_factory_setter;
    if (!mdns_client_->StartListening()) {
      VLOG(1) << "Failed to start MDnsClient";
      Send(new LocalDiscoveryHostMsg_Error());
      return;
    }
  }

  service_discovery_client_.reset(
      new local_discovery::ServiceDiscoveryClientImpl(mdns_client_.get()));
}

bool ServiceDiscoveryMessageHandler::InitializeThread() {
  if (discovery_task_runner_)
    return true;
  if (discovery_thread_)
    return false;
  utility_task_runner_ = base::MessageLoop::current()->message_loop_proxy();
  discovery_thread_.reset(new base::Thread("ServiceDiscoveryThread"));
  base::Thread::Options thread_options(base::MessageLoop::TYPE_IO, 0);
  if (discovery_thread_->StartWithOptions(thread_options)) {
    discovery_task_runner_ = discovery_thread_->message_loop_proxy();
    discovery_task_runner_->PostTask(FROM_HERE,
        base::Bind(&ServiceDiscoveryMessageHandler::InitializeMdns,
                    base::Unretained(this)));
  }
  return discovery_task_runner_ != NULL;
}

bool ServiceDiscoveryMessageHandler::OnMessageReceived(
    const IPC::Message& message) {
  bool handled = true;
  IPC_BEGIN_MESSAGE_MAP(ServiceDiscoveryMessageHandler, message)
#if defined(OS_POSIX)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_SetSockets, OnSetSockets)
#endif  // OS_POSIX
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_StartWatcher, OnStartWatcher)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_DiscoverServices, OnDiscoverServices)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_DestroyWatcher, OnDestroyWatcher)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_ResolveService, OnResolveService)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_DestroyResolver, OnDestroyResolver)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_ResolveLocalDomain,
                        OnResolveLocalDomain)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_DestroyLocalDomainResolver,
                        OnDestroyLocalDomainResolver)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_ShutdownLocalDiscovery,
                        ShutdownLocalDiscovery)
    IPC_MESSAGE_UNHANDLED(handled = false)
  IPC_END_MESSAGE_MAP()
  return handled;
}

void ServiceDiscoveryMessageHandler::PostTask(
    const tracked_objects::Location& from_here,
    const base::Closure& task) {
  if (!InitializeThread())
    return;
  discovery_task_runner_->PostTask(from_here, task);
}

#if defined(OS_POSIX)
void ServiceDiscoveryMessageHandler::OnSetSockets(
    const base::FileDescriptor& socket_v4,
    const base::FileDescriptor& socket_v6) {
  g_local_discovery_socket_factory.Get().SetSockets(socket_v4.fd, socket_v6.fd);
}
#endif  // OS_POSIX

void ServiceDiscoveryMessageHandler::OnStartWatcher(
    uint64 id,
    const std::string& service_type) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::StartWatcher,
                      base::Unretained(this), id, service_type));
}

void ServiceDiscoveryMessageHandler::OnDiscoverServices(uint64 id,
                                                        bool force_update) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::DiscoverServices,
                      base::Unretained(this), id, force_update));
}

void ServiceDiscoveryMessageHandler::OnDestroyWatcher(uint64 id) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::DestroyWatcher,
                      base::Unretained(this), id));
}

void ServiceDiscoveryMessageHandler::OnResolveService(
    uint64 id,
    const std::string& service_name) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::ResolveService,
                      base::Unretained(this), id, service_name));
}

void ServiceDiscoveryMessageHandler::OnDestroyResolver(uint64 id) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::DestroyResolver,
                      base::Unretained(this), id));
}

void ServiceDiscoveryMessageHandler::OnResolveLocalDomain(
    uint64 id, const std::string& domain,
    net::AddressFamily address_family) {
    PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::ResolveLocalDomain,
                      base::Unretained(this), id, domain, address_family));
}

void ServiceDiscoveryMessageHandler::OnDestroyLocalDomainResolver(uint64 id) {
  PostTask(FROM_HERE,
           base::Bind(
               &ServiceDiscoveryMessageHandler::DestroyLocalDomainResolver,
               base::Unretained(this), id));
}

void ServiceDiscoveryMessageHandler::StartWatcher(
    uint64 id,
    const std::string& service_type) {
  VLOG(1) << "StartWatcher, id=" << id << ", type=" << service_type;
  if (!service_discovery_client_)
    return;
  DCHECK(!ContainsKey(service_watchers_, id));
  scoped_ptr<ServiceWatcher> watcher(
      service_discovery_client_->CreateServiceWatcher(
          service_type,
          base::Bind(&ServiceDiscoveryMessageHandler::OnServiceUpdated,
                     base::Unretained(this), id)));
  watcher->Start();
  service_watchers_[id].reset(watcher.release());
}

void ServiceDiscoveryMessageHandler::DiscoverServices(uint64 id,
                                                      bool force_update) {
  VLOG(1) << "DiscoverServices, id=" << id;
  if (!service_discovery_client_)
    return;
  DCHECK(ContainsKey(service_watchers_, id));
  service_watchers_[id]->DiscoverNewServices(force_update);
}

void ServiceDiscoveryMessageHandler::DestroyWatcher(uint64 id) {
  VLOG(1) << "DestoryWatcher, id=" << id;
  if (!service_discovery_client_)
    return;
  service_watchers_.erase(id);
}

void ServiceDiscoveryMessageHandler::ResolveService(
    uint64 id,
    const std::string& service_name) {
  VLOG(1) << "ResolveService, id=" << id << ", name=" << service_name;
  if (!service_discovery_client_)
    return;
  DCHECK(!ContainsKey(service_resolvers_, id));
  scoped_ptr<ServiceResolver> resolver(
      service_discovery_client_->CreateServiceResolver(
          service_name,
          base::Bind(&ServiceDiscoveryMessageHandler::OnServiceResolved,
                     base::Unretained(this), id)));
  resolver->StartResolving();
  service_resolvers_[id].reset(resolver.release());
}

void ServiceDiscoveryMessageHandler::DestroyResolver(uint64 id) {
  VLOG(1) << "DestroyResolver, id=" << id;
  if (!service_discovery_client_)
    return;
  service_resolvers_.erase(id);
}

void ServiceDiscoveryMessageHandler::ResolveLocalDomain(
    uint64 id,
    const std::string& domain,
    net::AddressFamily address_family) {
  VLOG(1) << "ResolveLocalDomain, id=" << id << ", domain=" << domain;
  if (!service_discovery_client_)
    return;
  DCHECK(!ContainsKey(local_domain_resolvers_, id));
  scoped_ptr<LocalDomainResolver> resolver(
      service_discovery_client_->CreateLocalDomainResolver(
          domain, address_family,
          base::Bind(&ServiceDiscoveryMessageHandler::OnLocalDomainResolved,
                     base::Unretained(this), id)));
  resolver->Start();
  local_domain_resolvers_[id].reset(resolver.release());
}

void ServiceDiscoveryMessageHandler::DestroyLocalDomainResolver(uint64 id) {
  VLOG(1) << "DestroyLocalDomainResolver, id=" << id;
  if (!service_discovery_client_)
    return;
  local_domain_resolvers_.erase(id);
}

void ServiceDiscoveryMessageHandler::ShutdownLocalDiscovery() {
  discovery_task_runner_->PostTask(
      FROM_HERE,
      base::Bind(&ServiceDiscoveryMessageHandler::ShutdownOnIOThread,
                 base::Unretained(this)));

  // This will wait for message loop to drain, so ShutdownOnIOThread will
  // definitely be called.
  discovery_thread_.reset();
}

void ServiceDiscoveryMessageHandler::ShutdownOnIOThread() {
  VLOG(1) << "ShutdownLocalDiscovery";
  service_watchers_.clear();
  service_resolvers_.clear();
  local_domain_resolvers_.clear();
  service_discovery_client_.reset();
  mdns_client_.reset();
}

void ServiceDiscoveryMessageHandler::OnServiceUpdated(
    uint64 id,
    ServiceWatcher::UpdateType update,
    const std::string& name) {
  VLOG(1) << "OnServiceUpdated, id=" << id
          << ", status=" << WatcherUpdateToString(update) << ", name=" << name;
  DCHECK(service_discovery_client_);

  Send(new LocalDiscoveryHostMsg_WatcherCallback(id, update, name));
}

void ServiceDiscoveryMessageHandler::OnServiceResolved(
    uint64 id,
    ServiceResolver::RequestStatus status,
    const ServiceDescription& description) {
  VLOG(1) << "OnServiceResolved, id=" << id
          << ", status=" << ResolverStatusToString(status)
          << ", name=" << description.service_name;

  DCHECK(service_discovery_client_);
  Send(new LocalDiscoveryHostMsg_ResolverCallback(id, status, description));
}

void ServiceDiscoveryMessageHandler::OnLocalDomainResolved(
    uint64 id,
    bool success,
    const net::IPAddressNumber& address_ipv4,
    const net::IPAddressNumber& address_ipv6) {
  VLOG(1) << "OnLocalDomainResolved, id=" << id
          << ", IPv4=" << net::IPAddressToString(address_ipv4)
          << ", IPv6=" << net::IPAddressToString(address_ipv6);

  DCHECK(service_discovery_client_);
  Send(new LocalDiscoveryHostMsg_LocalDomainResolverCallback(
          id, success, address_ipv4, address_ipv6));
}

void ServiceDiscoveryMessageHandler::Send(IPC::Message* msg) {
  utility_task_runner_->PostTask(FROM_HERE,
                                 base::Bind(&SendHostMessageOnUtilityThread,
                                            msg));
}

}  // namespace local_discovery
<commit_msg>Fix NULL pointer exception when ServiceDiscoveryClientMdns is shutdown before the discovery thread is initialized.<commit_after>// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/utility/local_discovery/service_discovery_message_handler.h"

#include <algorithm>

#include "base/lazy_instance.h"
#include "chrome/common/local_discovery/local_discovery_messages.h"
#include "chrome/utility/local_discovery/service_discovery_client_impl.h"
#include "content/public/utility/utility_thread.h"
#include "net/socket/socket_descriptor.h"

namespace local_discovery {

namespace {

void ClosePlatformSocket(net::SocketDescriptor socket);

class SocketFactory : public net::PlatformSocketFactory {
 public:
  SocketFactory()
      : socket_v4_(net::kInvalidSocket),
        socket_v6_(net::kInvalidSocket) {
  }

  void SetSockets(net::SocketDescriptor socket_v4,
                  net::SocketDescriptor socket_v6) {
    Reset();
    socket_v4_ = socket_v4;
    socket_v6_ = socket_v6;
    VLOG(1) << "SetSockets: " << socket_v4_ << " " << socket_v6_;
  }

  void Reset() {
    if (socket_v4_ != net::kInvalidSocket) {
      ClosePlatformSocket(socket_v4_);
      socket_v4_ = net::kInvalidSocket;
    }
    if (socket_v6_ != net::kInvalidSocket) {
      ClosePlatformSocket(socket_v6_);
      socket_v6_ = net::kInvalidSocket;
    }
  }

  virtual ~SocketFactory() {
    Reset();
  }

 protected:
  virtual net::SocketDescriptor CreateSocket(int family, int type,
                                             int protocol) OVERRIDE {
    net::SocketDescriptor result = net::kInvalidSocket;
    if (type != SOCK_DGRAM) {
      NOTREACHED();
    } else if (family == AF_INET) {
      std::swap(result, socket_v4_);
    } else if (family == AF_INET6) {
      std::swap(result, socket_v6_);
    }
    return result;
  }

 private:
  net::SocketDescriptor socket_v4_;
  net::SocketDescriptor socket_v6_;

  DISALLOW_COPY_AND_ASSIGN(SocketFactory);
};

base::LazyInstance<SocketFactory>
    g_local_discovery_socket_factory = LAZY_INSTANCE_INITIALIZER;

class ScopedSocketFactorySetter {
 public:
  ScopedSocketFactorySetter() {
    net::PlatformSocketFactory::SetInstance(
        &g_local_discovery_socket_factory.Get());
  }

  ~ScopedSocketFactorySetter() {
    net::PlatformSocketFactory::SetInstance(NULL);
    g_local_discovery_socket_factory.Get().Reset();
  }

 private:
  DISALLOW_COPY_AND_ASSIGN(ScopedSocketFactorySetter);
};

#if defined(OS_WIN)

void ClosePlatformSocket(net::SocketDescriptor socket) {
  ::closesocket(socket);
}

void StaticInitializeSocketFactory() {
  g_local_discovery_socket_factory.Get().SetSockets(
      net::CreatePlatformSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP),
      net::CreatePlatformSocket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP));
}

#else  // OS_WIN

void ClosePlatformSocket(net::SocketDescriptor socket) {
  ::close(socket);
}

void StaticInitializeSocketFactory() {
}

#endif  // OS_WIN

void SendHostMessageOnUtilityThread(IPC::Message* msg) {
  content::UtilityThread::Get()->Send(msg);
}

std::string WatcherUpdateToString(ServiceWatcher::UpdateType update) {
  switch (update) {
    case ServiceWatcher::UPDATE_ADDED:
      return "UPDATE_ADDED";
    case ServiceWatcher::UPDATE_CHANGED:
      return "UPDATE_CHANGED";
    case ServiceWatcher::UPDATE_REMOVED:
      return "UPDATE_REMOVED";
    case ServiceWatcher::UPDATE_INVALIDATED:
      return "UPDATE_INVALIDATED";
  }
  return "Unknown Update";
}

std::string ResolverStatusToString(ServiceResolver::RequestStatus status) {
  switch (status) {
    case ServiceResolver::STATUS_SUCCESS:
      return "STATUS_SUCESS";
    case ServiceResolver::STATUS_REQUEST_TIMEOUT:
      return "STATUS_REQUEST_TIMEOUT";
    case ServiceResolver::STATUS_KNOWN_NONEXISTENT:
      return "STATUS_KNOWN_NONEXISTENT";
  }
  return "Unknown Status";
}

}  // namespace

ServiceDiscoveryMessageHandler::ServiceDiscoveryMessageHandler() {
}

ServiceDiscoveryMessageHandler::~ServiceDiscoveryMessageHandler() {
  DCHECK(!discovery_thread_);
}

void ServiceDiscoveryMessageHandler::PreSandboxStartup() {
  StaticInitializeSocketFactory();
}

void ServiceDiscoveryMessageHandler::InitializeMdns() {
  if (service_discovery_client_ || mdns_client_)
    return;

  mdns_client_ = net::MDnsClient::CreateDefault();
  {
    // Temporarily redirect network code to use pre-created sockets.
    ScopedSocketFactorySetter socket_factory_setter;
    if (!mdns_client_->StartListening()) {
      VLOG(1) << "Failed to start MDnsClient";
      Send(new LocalDiscoveryHostMsg_Error());
      return;
    }
  }

  service_discovery_client_.reset(
      new local_discovery::ServiceDiscoveryClientImpl(mdns_client_.get()));
}

bool ServiceDiscoveryMessageHandler::InitializeThread() {
  if (discovery_task_runner_)
    return true;
  if (discovery_thread_)
    return false;
  utility_task_runner_ = base::MessageLoop::current()->message_loop_proxy();
  discovery_thread_.reset(new base::Thread("ServiceDiscoveryThread"));
  base::Thread::Options thread_options(base::MessageLoop::TYPE_IO, 0);
  if (discovery_thread_->StartWithOptions(thread_options)) {
    discovery_task_runner_ = discovery_thread_->message_loop_proxy();
    discovery_task_runner_->PostTask(FROM_HERE,
        base::Bind(&ServiceDiscoveryMessageHandler::InitializeMdns,
                    base::Unretained(this)));
  }
  return discovery_task_runner_ != NULL;
}

bool ServiceDiscoveryMessageHandler::OnMessageReceived(
    const IPC::Message& message) {
  bool handled = true;
  IPC_BEGIN_MESSAGE_MAP(ServiceDiscoveryMessageHandler, message)
#if defined(OS_POSIX)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_SetSockets, OnSetSockets)
#endif  // OS_POSIX
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_StartWatcher, OnStartWatcher)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_DiscoverServices, OnDiscoverServices)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_DestroyWatcher, OnDestroyWatcher)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_ResolveService, OnResolveService)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_DestroyResolver, OnDestroyResolver)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_ResolveLocalDomain,
                        OnResolveLocalDomain)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_DestroyLocalDomainResolver,
                        OnDestroyLocalDomainResolver)
    IPC_MESSAGE_HANDLER(LocalDiscoveryMsg_ShutdownLocalDiscovery,
                        ShutdownLocalDiscovery)
    IPC_MESSAGE_UNHANDLED(handled = false)
  IPC_END_MESSAGE_MAP()
  return handled;
}

void ServiceDiscoveryMessageHandler::PostTask(
    const tracked_objects::Location& from_here,
    const base::Closure& task) {
  if (!InitializeThread())
    return;
  discovery_task_runner_->PostTask(from_here, task);
}

#if defined(OS_POSIX)
void ServiceDiscoveryMessageHandler::OnSetSockets(
    const base::FileDescriptor& socket_v4,
    const base::FileDescriptor& socket_v6) {
  g_local_discovery_socket_factory.Get().SetSockets(socket_v4.fd, socket_v6.fd);
}
#endif  // OS_POSIX

void ServiceDiscoveryMessageHandler::OnStartWatcher(
    uint64 id,
    const std::string& service_type) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::StartWatcher,
                      base::Unretained(this), id, service_type));
}

void ServiceDiscoveryMessageHandler::OnDiscoverServices(uint64 id,
                                                        bool force_update) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::DiscoverServices,
                      base::Unretained(this), id, force_update));
}

void ServiceDiscoveryMessageHandler::OnDestroyWatcher(uint64 id) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::DestroyWatcher,
                      base::Unretained(this), id));
}

void ServiceDiscoveryMessageHandler::OnResolveService(
    uint64 id,
    const std::string& service_name) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::ResolveService,
                      base::Unretained(this), id, service_name));
}

void ServiceDiscoveryMessageHandler::OnDestroyResolver(uint64 id) {
  PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::DestroyResolver,
                      base::Unretained(this), id));
}

void ServiceDiscoveryMessageHandler::OnResolveLocalDomain(
    uint64 id, const std::string& domain,
    net::AddressFamily address_family) {
    PostTask(FROM_HERE,
           base::Bind(&ServiceDiscoveryMessageHandler::ResolveLocalDomain,
                      base::Unretained(this), id, domain, address_family));
}

void ServiceDiscoveryMessageHandler::OnDestroyLocalDomainResolver(uint64 id) {
  PostTask(FROM_HERE,
           base::Bind(
               &ServiceDiscoveryMessageHandler::DestroyLocalDomainResolver,
               base::Unretained(this), id));
}

void ServiceDiscoveryMessageHandler::StartWatcher(
    uint64 id,
    const std::string& service_type) {
  VLOG(1) << "StartWatcher, id=" << id << ", type=" << service_type;
  if (!service_discovery_client_)
    return;
  DCHECK(!ContainsKey(service_watchers_, id));
  scoped_ptr<ServiceWatcher> watcher(
      service_discovery_client_->CreateServiceWatcher(
          service_type,
          base::Bind(&ServiceDiscoveryMessageHandler::OnServiceUpdated,
                     base::Unretained(this), id)));
  watcher->Start();
  service_watchers_[id].reset(watcher.release());
}

void ServiceDiscoveryMessageHandler::DiscoverServices(uint64 id,
                                                      bool force_update) {
  VLOG(1) << "DiscoverServices, id=" << id;
  if (!service_discovery_client_)
    return;
  DCHECK(ContainsKey(service_watchers_, id));
  service_watchers_[id]->DiscoverNewServices(force_update);
}

void ServiceDiscoveryMessageHandler::DestroyWatcher(uint64 id) {
  VLOG(1) << "DestoryWatcher, id=" << id;
  if (!service_discovery_client_)
    return;
  service_watchers_.erase(id);
}

void ServiceDiscoveryMessageHandler::ResolveService(
    uint64 id,
    const std::string& service_name) {
  VLOG(1) << "ResolveService, id=" << id << ", name=" << service_name;
  if (!service_discovery_client_)
    return;
  DCHECK(!ContainsKey(service_resolvers_, id));
  scoped_ptr<ServiceResolver> resolver(
      service_discovery_client_->CreateServiceResolver(
          service_name,
          base::Bind(&ServiceDiscoveryMessageHandler::OnServiceResolved,
                     base::Unretained(this), id)));
  resolver->StartResolving();
  service_resolvers_[id].reset(resolver.release());
}

void ServiceDiscoveryMessageHandler::DestroyResolver(uint64 id) {
  VLOG(1) << "DestroyResolver, id=" << id;
  if (!service_discovery_client_)
    return;
  service_resolvers_.erase(id);
}

void ServiceDiscoveryMessageHandler::ResolveLocalDomain(
    uint64 id,
    const std::string& domain,
    net::AddressFamily address_family) {
  VLOG(1) << "ResolveLocalDomain, id=" << id << ", domain=" << domain;
  if (!service_discovery_client_)
    return;
  DCHECK(!ContainsKey(local_domain_resolvers_, id));
  scoped_ptr<LocalDomainResolver> resolver(
      service_discovery_client_->CreateLocalDomainResolver(
          domain, address_family,
          base::Bind(&ServiceDiscoveryMessageHandler::OnLocalDomainResolved,
                     base::Unretained(this), id)));
  resolver->Start();
  local_domain_resolvers_[id].reset(resolver.release());
}

void ServiceDiscoveryMessageHandler::DestroyLocalDomainResolver(uint64 id) {
  VLOG(1) << "DestroyLocalDomainResolver, id=" << id;
  if (!service_discovery_client_)
    return;
  local_domain_resolvers_.erase(id);
}

void ServiceDiscoveryMessageHandler::ShutdownLocalDiscovery() {
  if (!discovery_task_runner_)
    return;

  discovery_task_runner_->PostTask(
      FROM_HERE,
      base::Bind(&ServiceDiscoveryMessageHandler::ShutdownOnIOThread,
                 base::Unretained(this)));

  // This will wait for message loop to drain, so ShutdownOnIOThread will
  // definitely be called.
  discovery_thread_.reset();
}

void ServiceDiscoveryMessageHandler::ShutdownOnIOThread() {
  VLOG(1) << "ShutdownLocalDiscovery";
  service_watchers_.clear();
  service_resolvers_.clear();
  local_domain_resolvers_.clear();
  service_discovery_client_.reset();
  mdns_client_.reset();
}

void ServiceDiscoveryMessageHandler::OnServiceUpdated(
    uint64 id,
    ServiceWatcher::UpdateType update,
    const std::string& name) {
  VLOG(1) << "OnServiceUpdated, id=" << id
          << ", status=" << WatcherUpdateToString(update) << ", name=" << name;
  DCHECK(service_discovery_client_);

  Send(new LocalDiscoveryHostMsg_WatcherCallback(id, update, name));
}

void ServiceDiscoveryMessageHandler::OnServiceResolved(
    uint64 id,
    ServiceResolver::RequestStatus status,
    const ServiceDescription& description) {
  VLOG(1) << "OnServiceResolved, id=" << id
          << ", status=" << ResolverStatusToString(status)
          << ", name=" << description.service_name;

  DCHECK(service_discovery_client_);
  Send(new LocalDiscoveryHostMsg_ResolverCallback(id, status, description));
}

void ServiceDiscoveryMessageHandler::OnLocalDomainResolved(
    uint64 id,
    bool success,
    const net::IPAddressNumber& address_ipv4,
    const net::IPAddressNumber& address_ipv6) {
  VLOG(1) << "OnLocalDomainResolved, id=" << id
          << ", IPv4=" << net::IPAddressToString(address_ipv4)
          << ", IPv6=" << net::IPAddressToString(address_ipv6);

  DCHECK(service_discovery_client_);
  Send(new LocalDiscoveryHostMsg_LocalDomainResolverCallback(
          id, success, address_ipv4, address_ipv6));
}

void ServiceDiscoveryMessageHandler::Send(IPC::Message* msg) {
  utility_task_runner_->PostTask(FROM_HERE,
                                 base::Bind(&SendHostMessageOnUtilityThread,
                                            msg));
}

}  // namespace local_discovery
<|endoftext|>


<commit_before>/// \file
/// \ingroup tutorial_vecops
/// \notebook -nodraw
/// In this tutorial we learn how the RVec class can be used to
/// express easily mathematical operations involving arrays and scalars.
///
/// \macro_code
///
/// \date May 2018
/// \author Danilo Piparo

using namespace ROOT::VecOps;

void vo002_VectorCalculations()
{

   // Operations on RVec instances are made to be *fast* (vectorisation is exploited)
   // and easy to use.
   RVec<float> v1{1., 2., 3.};
   RVec<float> v2{4., 5., 6.};

   // Arithmetic operations are to be intended on pairs of elements with identical index
   auto v_sum = v1 + v2;
   auto v_mul = v1 * v2;

   // Easy to inspect:
   std::cout << "v1 = " << v1 << "\n"
             << "v2 = " << v2 << "\n"
             << "v1 + v2 = " << v_sum << "\n"
             << "v1 * v2 = " << v_mul << std::endl;

   // It's also possible to mix scalars and RVecs
   auto v_diff_s_0 = v1 - 2;
   auto v_diff_s_1 = 2 - v1;
   auto v_div_s_0 = v1 / 2.;
   auto v_div_s_1 = 2. / v1;

   std::cout << v1 << " - 2 = " << v_diff_s_0 << "\n"
             << "2 - " << v1 << " = " << v_diff_s_1 << "\n"
             << v1 << " / 2 = " << v_div_s_0 << "\n"
             << "2 / " << v1 << " = " << v_div_s_1 << std::endl;

   // Dot product and the extraction of quantities such as Mean, Min and Max
   // are also easy to express (see here for the full list:
   // https://root.cern.ch/doc/master/namespaceROOT_1_1VecOps.html)
   auto v1_mean = Mean(v1);
   auto v1_dot_v2 = Dot(v1, v2);

   std::cout << "Mean of " << v1 << " is " << v1_mean << "\n"
             << "Dot product of " << v1 << " and " << v2 << " is " << v1_dot_v2 << std::endl;

   // Most used mathematical functions are supported
   auto v_exp = exp(v1);
   auto v_log = log(v1);
   auto v_sin = sin(v1);

   std::cout << "exp(" << v1 << ") = " << v_exp << "\n"
             << "log(" << v1 << ") = " << v_log << "\n"
             << "sin(" << v1 << ") = " << v_sin << std::endl;

   // Even an optimised version of the functions is available
   auto v_fast_exp = fast_exp(v1);
   auto v_fast_log = fast_log(v1);
   auto v_fast_sin = fast_sin(v1);

   std::cout << "fast_exp(" << v1 << ") = " << v_fast_exp << "\n"
             << "fast_log(" << v1 << ") = " << v_fast_log << "\n"
             << "fast_sin(" << v1 << ") = " << v_fast_sin << std::endl;

   // It may happen that a custom operation needs to be applied to the RVec.
   // In this case, the Map utitlity can be used:
   auto v_transf = Map(v1, [](double x) { return x * 2 / 3; });

   std::cout << "Applying [](double x){return x * 2 / 3;} to " << v1 << " leads to " << v_transf << "\n";
}
<commit_msg>[VecOps] Do not invoke vdt functions if vdt is disabled<commit_after>/// \file
/// \ingroup tutorial_vecops
/// \notebook -nodraw
/// In this tutorial we learn how the RVec class can be used to
/// express easily mathematical operations involving arrays and scalars.
///
/// \macro_code
///
/// \date May 2018
/// \author Danilo Piparo

using namespace ROOT::VecOps;

void vo002_VectorCalculations()
{

   // Operations on RVec instances are made to be *fast* (vectorisation is exploited)
   // and easy to use.
   RVec<float> v1{1., 2., 3.};
   RVec<float> v2{4., 5., 6.};

   // Arithmetic operations are to be intended on pairs of elements with identical index
   auto v_sum = v1 + v2;
   auto v_mul = v1 * v2;

   // Easy to inspect:
   std::cout << "v1 = " << v1 << "\n"
             << "v2 = " << v2 << "\n"
             << "v1 + v2 = " << v_sum << "\n"
             << "v1 * v2 = " << v_mul << std::endl;

   // It's also possible to mix scalars and RVecs
   auto v_diff_s_0 = v1 - 2;
   auto v_diff_s_1 = 2 - v1;
   auto v_div_s_0 = v1 / 2.;
   auto v_div_s_1 = 2. / v1;

   std::cout << v1 << " - 2 = " << v_diff_s_0 << "\n"
             << "2 - " << v1 << " = " << v_diff_s_1 << "\n"
             << v1 << " / 2 = " << v_div_s_0 << "\n"
             << "2 / " << v1 << " = " << v_div_s_1 << std::endl;

   // Dot product and the extraction of quantities such as Mean, Min and Max
   // are also easy to express (see here for the full list:
   // https://root.cern.ch/doc/master/namespaceROOT_1_1VecOps.html)
   auto v1_mean = Mean(v1);
   auto v1_dot_v2 = Dot(v1, v2);

   std::cout << "Mean of " << v1 << " is " << v1_mean << "\n"
             << "Dot product of " << v1 << " and " << v2 << " is " << v1_dot_v2 << std::endl;

   // Most used mathematical functions are supported
   auto v_exp = exp(v1);
   auto v_log = log(v1);
   auto v_sin = sin(v1);

   std::cout << "exp(" << v1 << ") = " << v_exp << "\n"
             << "log(" << v1 << ") = " << v_log << "\n"
             << "sin(" << v1 << ") = " << v_sin << std::endl;

   // Even an optimised version of the functions is available
   // provided that VDT is not disabled during the configuration
#ifdef R__HAS_VDT
   auto v_fast_exp = fast_exp(v1);
   auto v_fast_log = fast_log(v1);
   auto v_fast_sin = fast_sin(v1);

   std::cout << "fast_exp(" << v1 << ") = " << v_fast_exp << "\n"
             << "fast_log(" << v1 << ") = " << v_fast_log << "\n"
             << "fast_sin(" << v1 << ") = " << v_fast_sin << std::endl;

   // It may happen that a custom operation needs to be applied to the RVec.
   // In this case, the Map utitlity can be used:
   auto v_transf = Map(v1, [](double x) { return x * 2 / 3; });

   std::cout << "Applying [](double x){return x * 2 / 3;} to " << v1 << " leads to " << v_transf << "\n";
#endif
}
<|endoftext|>


<commit_before>#pragma once

#include <eosio/chain/transaction_metadata.hpp>
#include <eosio/chain/trace.hpp>
#include <eosio/chain/block_state.hpp>
#include <eosio/chain/exceptions.hpp>

#include <boost/multi_index_container.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>

namespace fc {
  inline std::size_t hash_value( const fc::sha256& v ) {
     return v._hash[3];
  }
}

namespace eosio { namespace chain {

using namespace boost::multi_index;

enum class trx_enum_type {
   unknown = 0,
   persisted = 1,
   forked = 2,
   aborted = 3,
   incoming = 4
};

using next_func_t = std::function<void(const fc::static_variant<fc::exception_ptr, transaction_trace_ptr>&)>;

struct unapplied_transaction {
   const transaction_metadata_ptr trx_meta;
   const fc::time_point           expiry;
   trx_enum_type                  trx_type = trx_enum_type::unknown;
   next_func_t                    next;

   const transaction_id_type& id()const { return trx_meta->id(); }

   unapplied_transaction(const unapplied_transaction&) = delete;
   unapplied_transaction() = delete;
   unapplied_transaction& operator=(const unapplied_transaction&) = delete;
   unapplied_transaction(unapplied_transaction&&) = default;
};

/**
 * Track unapplied transactions for persisted, forked blocks, and aborted blocks.
 * Persisted are first so that they can be applied in each block until expired.
 */
class unapplied_transaction_queue {
public:
   enum class process_mode {
      non_speculative,           // HEAD, READ_ONLY, IRREVERSIBLE
      speculative_non_producer,  // will never produce
      speculative_producer       // can produce
   };

private:
   struct by_trx_id;
   struct by_type;
   struct by_expiry;

   typedef multi_index_container< unapplied_transaction,
      indexed_by<
         hashed_unique< tag<by_trx_id>,
               const_mem_fun<unapplied_transaction, const transaction_id_type&, &unapplied_transaction::id>
         >,
         ordered_non_unique< tag<by_type>, member<unapplied_transaction, trx_enum_type, &unapplied_transaction::trx_type> >,
         ordered_non_unique< tag<by_expiry>, member<unapplied_transaction, const fc::time_point, &unapplied_transaction::expiry> >
      >
   > unapplied_trx_queue_type;

   unapplied_trx_queue_type queue;
   process_mode mode = process_mode::speculative_producer;

public:

   void set_mode( process_mode new_mode ) {
      if( new_mode != mode ) {
         FC_ASSERT( empty(), "set_mode, queue required to be empty" );
      }
      mode = new_mode;
   }

   bool empty() const {
      return queue.empty();
   }

   size_t size() const {
      return queue.size();
   }

   void clear() {
      queue.clear();
   }

   bool contains_persisted()const {
      return queue.get<by_type>().find( trx_enum_type::persisted ) != queue.get<by_type>().end();
   }

   bool is_persisted(const transaction_metadata_ptr& trx)const {
      auto itr = queue.get<by_trx_id>().find( trx->id() );
      if( itr == queue.get<by_trx_id>().end() ) return false;
      return itr->trx_type == trx_enum_type::persisted;
   }

   transaction_metadata_ptr get_trx( const transaction_id_type& id ) const {
      auto itr = queue.get<by_trx_id>().find( id );
      if( itr == queue.get<by_trx_id>().end() ) return {};
      return itr->trx_meta;
   }

   template <typename Func>
   bool clear_expired( const time_point& pending_block_time, const time_point& deadline, Func&& callback ) {
      auto& persisted_by_expiry = queue.get<by_expiry>();
      while( !persisted_by_expiry.empty() ) {
         const auto& itr = persisted_by_expiry.begin();
         if( itr->expiry <= pending_block_time ) {
            break;
         }
         if( deadline <= fc::time_point::now() ) {
            return false;
         }
         callback( itr->id(), itr->trx_type );
         if( itr->next ) {
            itr->next( std::static_pointer_cast<fc::exception>(
                  std::make_shared<expired_tx_exception>(
                        FC_LOG_MESSAGE( error, "expired transaction ${id}, expiration ${e}, block time ${bt}",
                                        ("id", itr->id())("e", itr->trx_meta->packed_trx()->expiration())
                                        ("bt", pending_block_time) ) ) ) );
         }

         persisted_by_expiry.erase( itr );
      }
      return true;
   }

   void clear_applied( const block_state_ptr& bs ) {
      if( empty() ) return;
      auto& idx = queue.get<by_trx_id>();
      for( const auto& receipt : bs->block->transactions ) {
         if( receipt.trx.contains<packed_transaction>() ) {
            const auto& pt = receipt.trx.get<packed_transaction>();
            auto itr = queue.get<by_trx_id>().find( pt.id() );
            if( itr != queue.get<by_trx_id>().end() ) {
               if( itr->trx_type != trx_enum_type::persisted ) {
                  idx.erase( pt.id() );
               }
            }
         }
      }
   }

   void add_forked( const branch_type& forked_branch ) {
      if( mode == process_mode::non_speculative || mode == process_mode::speculative_non_producer ) return;
      // forked_branch is in reverse order
      for( auto ritr = forked_branch.rbegin(), rend = forked_branch.rend(); ritr != rend; ++ritr ) {
         const block_state_ptr& bsptr = *ritr;
         for( auto itr = bsptr->trxs_metas().begin(), end = bsptr->trxs_metas().end(); itr != end; ++itr ) {
            const auto& trx = *itr;
            fc::time_point expiry = trx->packed_trx()->expiration();
            queue.insert( { trx, expiry, trx_enum_type::forked } );
         }
      }
   }

   void add_aborted( std::vector<transaction_metadata_ptr> aborted_trxs ) {
      if( mode == process_mode::non_speculative || mode == process_mode::speculative_non_producer ) return;
      for( auto& trx : aborted_trxs ) {
         fc::time_point expiry = trx->packed_trx()->expiration();
         queue.insert( { std::move( trx ), expiry, trx_enum_type::aborted } );
      }
   }

   void add_persisted( const transaction_metadata_ptr& trx ) {
      if( mode == process_mode::non_speculative ) return;
      auto itr = queue.get<by_trx_id>().find( trx->id() );
      if( itr == queue.get<by_trx_id>().end() ) {
         fc::time_point expiry = trx->packed_trx()->expiration();
         queue.insert( { trx, expiry, trx_enum_type::persisted } );
      } else if( itr->trx_type != trx_enum_type::persisted ) {
         queue.get<by_trx_id>().modify( itr, [](auto& un){
            un.trx_type = trx_enum_type::persisted;
         } );
      }
   }

   void add_incoming( const transaction_metadata_ptr& trx, bool persist_until_expired ) {
      auto itr = queue.get<by_trx_id>().find( trx->id() );
      if( itr == queue.get<by_trx_id>().end() ) {
         fc::time_point expiry = trx->packed_trx()->expiration();
         queue.insert( { trx, expiry, persist_until_expired ? trx_enum_type::persisted : trx_enum_type::incoming } );
      } else if( persist_until_expired && itr->trx_type != trx_enum_type::persisted ) {
         queue.get<by_trx_id>().modify( itr, [](auto& un){
            un.trx_type = trx_enum_type::persisted;
         } );
      }
   }

   using iterator = unapplied_trx_queue_type::index<by_type>::type::iterator;

   iterator begin() { return queue.get<by_type>().begin(); }
   iterator end() { return queue.get<by_type>().end(); }

   iterator persisted_begin() { return queue.get<by_type>().lower_bound( trx_enum_type::persisted ); }
   iterator persisted_end() { return queue.get<by_type>().upper_bound( trx_enum_type::persisted ); }

   iterator incoming_begin() { return queue.get<by_type>().lower_bound( trx_enum_type::incoming ); }
   iterator incoming_end() { return queue.get<by_type>().upper_bound( trx_enum_type::incoming ); }

   iterator erase( iterator itr ) { return queue.get<by_type>().erase( itr ); }

};

} } //eosio::chain
<commit_msg>Add erase and incoming_size methods<commit_after>#pragma once

#include <eosio/chain/transaction_metadata.hpp>
#include <eosio/chain/trace.hpp>
#include <eosio/chain/block_state.hpp>
#include <eosio/chain/exceptions.hpp>

#include <boost/multi_index_container.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>

namespace fc {
  inline std::size_t hash_value( const fc::sha256& v ) {
     return v._hash[3];
  }
}

namespace eosio { namespace chain {

using namespace boost::multi_index;

enum class trx_enum_type {
   unknown = 0,
   persisted = 1,
   forked = 2,
   aborted = 3,
   incoming = 4
};

using next_func_t = std::function<void(const fc::static_variant<fc::exception_ptr, transaction_trace_ptr>&)>;

struct unapplied_transaction {
   const transaction_metadata_ptr trx_meta;
   const fc::time_point           expiry;
   trx_enum_type                  trx_type = trx_enum_type::unknown;
   next_func_t                    next;

   const transaction_id_type& id()const { return trx_meta->id(); }

   unapplied_transaction(const unapplied_transaction&) = delete;
   unapplied_transaction() = delete;
   unapplied_transaction& operator=(const unapplied_transaction&) = delete;
   unapplied_transaction(unapplied_transaction&&) = default;
};

/**
 * Track unapplied transactions for persisted, forked blocks, and aborted blocks.
 * Persisted are first so that they can be applied in each block until expired.
 */
class unapplied_transaction_queue {
public:
   enum class process_mode {
      non_speculative,           // HEAD, READ_ONLY, IRREVERSIBLE
      speculative_non_producer,  // will never produce
      speculative_producer       // can produce
   };

private:
   struct by_trx_id;
   struct by_type;
   struct by_expiry;

   typedef multi_index_container< unapplied_transaction,
      indexed_by<
         hashed_unique< tag<by_trx_id>,
               const_mem_fun<unapplied_transaction, const transaction_id_type&, &unapplied_transaction::id>
         >,
         ordered_non_unique< tag<by_type>, member<unapplied_transaction, trx_enum_type, &unapplied_transaction::trx_type> >,
         ordered_non_unique< tag<by_expiry>, member<unapplied_transaction, const fc::time_point, &unapplied_transaction::expiry> >
      >
   > unapplied_trx_queue_type;

   unapplied_trx_queue_type queue;
   process_mode mode = process_mode::speculative_producer;

public:

   void set_mode( process_mode new_mode ) {
      if( new_mode != mode ) {
         FC_ASSERT( empty(), "set_mode, queue required to be empty" );
      }
      mode = new_mode;
   }

   bool empty() const {
      return queue.empty();
   }

   size_t size() const {
      return queue.size();
   }

   void clear() {
      queue.clear();
   }

   bool contains_persisted()const {
      return queue.get<by_type>().find( trx_enum_type::persisted ) != queue.get<by_type>().end();
   }

   bool is_persisted(const transaction_metadata_ptr& trx)const {
      auto itr = queue.get<by_trx_id>().find( trx->id() );
      if( itr == queue.get<by_trx_id>().end() ) return false;
      return itr->trx_type == trx_enum_type::persisted;
   }

   size_t incoming_size()const {
      return queue.get<by_type>().count( trx_enum_type::incoming );
   }

   transaction_metadata_ptr get_trx( const transaction_id_type& id ) const {
      auto itr = queue.get<by_trx_id>().find( id );
      if( itr == queue.get<by_trx_id>().end() ) return {};
      return itr->trx_meta;
   }

   template <typename Func>
   bool clear_expired( const time_point& pending_block_time, const time_point& deadline, Func&& callback ) {
      auto& persisted_by_expiry = queue.get<by_expiry>();
      while( !persisted_by_expiry.empty() ) {
         const auto& itr = persisted_by_expiry.begin();
         if( itr->expiry <= pending_block_time ) {
            break;
         }
         if( deadline <= fc::time_point::now() ) {
            return false;
         }
         callback( itr->id(), itr->trx_type );
         if( itr->next ) {
            itr->next( std::static_pointer_cast<fc::exception>(
                  std::make_shared<expired_tx_exception>(
                        FC_LOG_MESSAGE( error, "expired transaction ${id}, expiration ${e}, block time ${bt}",
                                        ("id", itr->id())("e", itr->trx_meta->packed_trx()->expiration())
                                        ("bt", pending_block_time) ) ) ) );
         }

         persisted_by_expiry.erase( itr );
      }
      return true;
   }

   void clear_applied( const block_state_ptr& bs ) {
      if( empty() ) return;
      auto& idx = queue.get<by_trx_id>();
      for( const auto& receipt : bs->block->transactions ) {
         if( receipt.trx.contains<packed_transaction>() ) {
            const auto& pt = receipt.trx.get<packed_transaction>();
            auto itr = queue.get<by_trx_id>().find( pt.id() );
            if( itr != queue.get<by_trx_id>().end() ) {
               if( itr->trx_type != trx_enum_type::persisted ) {
                  idx.erase( pt.id() );
               }
            }
         }
      }
   }

   void add_forked( const branch_type& forked_branch ) {
      if( mode == process_mode::non_speculative || mode == process_mode::speculative_non_producer ) return;
      // forked_branch is in reverse order
      for( auto ritr = forked_branch.rbegin(), rend = forked_branch.rend(); ritr != rend; ++ritr ) {
         const block_state_ptr& bsptr = *ritr;
         for( auto itr = bsptr->trxs_metas().begin(), end = bsptr->trxs_metas().end(); itr != end; ++itr ) {
            const auto& trx = *itr;
            fc::time_point expiry = trx->packed_trx()->expiration();
            queue.insert( { trx, expiry, trx_enum_type::forked } );
         }
      }
   }

   void add_aborted( std::vector<transaction_metadata_ptr> aborted_trxs ) {
      if( mode == process_mode::non_speculative || mode == process_mode::speculative_non_producer ) return;
      for( auto& trx : aborted_trxs ) {
         fc::time_point expiry = trx->packed_trx()->expiration();
         queue.insert( { std::move( trx ), expiry, trx_enum_type::aborted } );
      }
   }

   void add_persisted( const transaction_metadata_ptr& trx ) {
      if( mode == process_mode::non_speculative ) return;
      auto itr = queue.get<by_trx_id>().find( trx->id() );
      if( itr == queue.get<by_trx_id>().end() ) {
         fc::time_point expiry = trx->packed_trx()->expiration();
         queue.insert( { trx, expiry, trx_enum_type::persisted } );
      } else if( itr->trx_type != trx_enum_type::persisted ) {
         queue.get<by_trx_id>().modify( itr, [](auto& un){
            un.trx_type = trx_enum_type::persisted;
         } );
      }
   }

   void add_incoming( const transaction_metadata_ptr& trx, bool persist_until_expired, next_func_t next ) {
      auto itr = queue.get<by_trx_id>().find( trx->id() );
      if( itr == queue.get<by_trx_id>().end() ) {
         fc::time_point expiry = trx->packed_trx()->expiration();
         queue.insert( { trx, expiry, persist_until_expired ? trx_enum_type::persisted : trx_enum_type::incoming, std::move( next ) } );
      } else if( (persist_until_expired && itr->trx_type != trx_enum_type::persisted) || (next && !itr->next) ) {
         queue.get<by_trx_id>().modify( itr, [next{std::move(next)}](auto& un) mutable {
            un.trx_type = trx_enum_type::persisted;
            if( next ) un.next = std::move( next );
         } );
      }
   }

   using iterator = unapplied_trx_queue_type::index<by_type>::type::iterator;

   iterator begin() { return queue.get<by_type>().begin(); }
   iterator end() { return queue.get<by_type>().end(); }

   iterator persisted_begin() { return queue.get<by_type>().lower_bound( trx_enum_type::persisted ); }
   iterator persisted_end() { return queue.get<by_type>().upper_bound( trx_enum_type::persisted ); }

   iterator incoming_begin() { return queue.get<by_type>().lower_bound( trx_enum_type::incoming ); }
   iterator incoming_end() { return queue.get<by_type>().upper_bound( trx_enum_type::incoming ); }

   iterator erase( iterator itr ) { return queue.get<by_type>().erase( itr ); }
   iterator erase( iterator b, iterator e ) { return queue.get<by_type>().erase( b, e ); }

};

} } //eosio::chain
<|endoftext|>


<commit_before>#include "gridmap.hpp"
#include "gui.hpp"
#include "oddlib/lvlarchive.hpp"
#include "renderer.hpp"
#include "oddlib/path.hpp"
#include "oddlib/bits_factory.hpp"
#include "logger.hpp"
#include <cassert>
#include "sdl_raii.hpp"
#include <algorithm> // min/max
#include <cmath>

Level::Level(GameData& gameData, IAudioController& /*audioController*/, FileSystem& fs)
    : mGameData(gameData), mFs(fs)
{

}

void Level::Update()
{
    if (mMap)
    {
        mMap->Update();
    }
}

void Level::Render(Renderer& rend, GuiContext& gui, int screenW, int screenH)
{
    RenderDebugPathSelection(rend, gui);
    if (mMap)
    {
        mMap->Render(rend, gui, screenW, screenH);
    }
}

void Level::RenderDebugPathSelection(Renderer& rend, GuiContext& gui)
{
    gui.next_window_pos = v2i(10, 300);
    gui_begin_window(&gui, "Paths", v2i(150, 400));

    static std::vector<std::pair<std::string, const GameData::PathEntry*>> items;
    if (items.empty())
    {

        for (const auto& pathGroup : mGameData.Paths())
        {
            for (const auto& path : pathGroup.second)
            {
                items.push_back(std::make_pair(pathGroup.first + " " + std::to_string(path.mPathChunkId), &path));
            }
        }
    }

    for (const auto& item : items)
    {
        if (gui_button(&gui, item.first.c_str()))
        {
            const GameData::PathEntry* entry = item.second;
            const std::string baseLvlNameUpper = item.first.substr(0, 2); // R1, MI etc
            std::string baseLvlNameLower = baseLvlNameUpper;

            // Open the LVL
            std::transform(baseLvlNameLower.begin(), baseLvlNameLower.end(), baseLvlNameLower.begin(), string_util::c_tolower);
            const std::string lvlName = baseLvlNameLower + ".lvl";

            LOG_INFO("Looking for LVL " << lvlName);

            auto chunkStream = mFs.ResourcePaths().OpenLvlFileChunkById(lvlName, baseLvlNameUpper + "PATH.BND", entry->mPathChunkId);
            if (chunkStream)
            {
                Oddlib::Path path(*chunkStream,
                    entry->mNumberOfCollisionItems,
                    entry->mObjectIndexTableOffset,
                    entry->mObjectDataOffset,
                    entry->mMapXSize,
                    entry->mMapYSize);
                mMap = std::make_unique<GridMap>(lvlName, path, mFs, rend);
            }
            else
            {
                LOG_ERROR("LVL or file in LVL not found");
            }
        }
    }

    gui_end_window(&gui);
}

GridScreen::GridScreen(const std::string& lvlName, const std::string& fileName, Renderer& rend)
    : mLvlName(lvlName),
      mFileName(fileName)
      , mTexHandle(0)
      , mRend(rend)
{

}

GridScreen::~GridScreen()
{
    mRend.destroyTexture(mTexHandle);
}

int GridScreen::getTexHandle(FileSystem& fs)
{
    if (!mTexHandle)
    {
        auto stream = fs.ResourcePaths().OpenLvlFileChunkByType(mLvlName, mFileName, Oddlib::MakeType('B', 'i', 't', 's'));
        if (stream)
        {
            auto bits = Oddlib::MakeBits(*stream);

            SDL_Surface* surf = bits->GetSurface();
            SDL_SurfacePtr converted;
            if (surf->format->format != SDL_PIXELFORMAT_RGB24)
            {
                converted.reset(SDL_ConvertSurfaceFormat(surf, SDL_PIXELFORMAT_RGB24, 0));
                surf = converted.get();
            }

#if 0
            SDL_SurfacePtr scaled;
            { // Scale the surface
                assert(surf->w == 640);
                assert(surf->h == 240);

                scaled.reset(SDL_CreateRGBSurface(0, 640, 480, 24, 0xFF, 0x00FF, 0x0000FF, 0));
                uint8_t *src = static_cast<uint8_t*>(surf->pixels);
                uint8_t *dst = static_cast<uint8_t*>(scaled->pixels);
                for (int y = 0; y < scaled->h; ++y) {
                    const int src_y = min(y, surf->h);
                    for (int x = 0; x < scaled->w; ++x) {
                        const int src_x = min(x, surf->w);

                        const int dst_ix = x*3 + scaled->pitch*y;
                        const int src_ix = src_x*3 + surf->pitch*src_y;

                        dst[dst_ix + 0] = src[src_ix + 0];
                        dst[dst_ix + 1] = src[src_ix + 1];
                        dst[dst_ix + 2] = src[src_ix + 2];
                    }
                }
            }
            mTexHandle = mRend.createTexture(GL_RGB, scaled->w, scaled->h, GL_RGB, GL_UNSIGNED_BYTE, scaled->pixels, false);
#else
            mTexHandle = mRend.createTexture(GL_RGB, surf->w, surf->h, GL_RGB, GL_UNSIGNED_BYTE, surf->pixels, false);
#endif
        }
    }
    return mTexHandle;
}

bool GridScreen::hasTexture() const
{
    bool onlySpaces = true;
    for (size_t i = 0; i < mFileName.size(); ++i) {
        if (mFileName[i] != ' ' && mFileName[i] != '\0')
        {
            onlySpaces = false;
            break;
        }
    }
    return !onlySpaces;
}

GridMap::GridMap(const std::string& lvlName, Oddlib::Path& path, FileSystem& fs, Renderer& rend)
    : mFs(fs), mLvlName(lvlName)
{
    mScreens.resize(path.XSize());
    for (auto& col : mScreens)
    {
        col.resize(path.YSize());
    }

    for (Uint32 x = 0; x < path.XSize(); x++)
    {
        for (Uint32 y = 0; y < path.YSize(); y++)
        {
            mScreens[x][y] = std::make_unique<GridScreen>(mLvlName, path.CameraFileName(x,y), rend);
        }
    }
}

void GridMap::Update()
{

}

void GridMap::Render(Renderer& rend, GuiContext& gui, int screenW, int screenH)
{
#if 0 // List of cams
    gui.next_window_pos = v2i(950, 50);
    gui_begin_window(&gui, "GridMap", v2i(200, 500));
    for (auto x = 0u; x < mScreens.size(); x++)
    {
        for (auto y = 0u; y < mScreens[0].size(); y++)
        {
            if (gui_button(&gui, gui_str(&gui, "cam_%i_%i|%s", (int)x, (int)y, mScreens[x][y]->FileName().c_str())))
            {
                mEditorScreenX = (int)x;
                mEditorScreenY = (int)y;
            }
        }
    }
    gui_end_window(&gui);

    if (mEditorScreenX >= static_cast<int>(mScreens.size()) || // Invalid x check
        (mEditorScreenX >= 0 && mEditorScreenY >= static_cast<int>(mScreens[mEditorScreenX].size()))) // Invalid y check
    {
        mEditorScreenX = mEditorScreenY = -1;
    }

    if (mEditorScreenX >= 0 && mEditorScreenY >= 0)
    {
        GridScreen *screen = mScreens[mEditorScreenX][mEditorScreenY].get();

        gui.next_window_pos = v2i(100, 100);

        V2i size = v2i(640, 480);
        gui_begin_window(&gui, "CAM", size);
        size = gui_window_client_size(&gui);

        rend.beginLayer(gui_layer(&gui));
        V2i pos = gui_turtle_pos(&gui);
        rend.drawQuad(screen->getTexHandle(mFs), 1.0f*pos.x, 1.0f*pos.y, 1.0f*size.x, 1.0f*size.y);
        rend.endLayer();

        gui_end_window(&gui);
    }
#else // Proper editor gui

    gui_begin_frame(&gui, "camArea", v2i(0, 0), v2i(screenW, screenH));
    rend.beginLayer(gui_layer(&gui));

    bool zoomChanged = false;
    if (gui.key_state[GUI_KEY_LCTRL] & GUI_KEYSTATE_DOWN_BIT)
    {
        mZoomLevel += gui.mouse_scroll;
        zoomChanged = (gui.mouse_scroll != 0);
    }
    const float zoomBase = 1.2f;
    const float oldZoomMul = std::pow(zoomBase, 1.f*mZoomLevel - gui.mouse_scroll);
    const float zoomMul = std::pow(zoomBase, 1.f*mZoomLevel);
    // Use oldZoom because gui_set_frame_scroll below doesn't change scrolling in current frame. Could be changed though.
    const V2i camSize = v2i((int)(1440*oldZoomMul), (int)(1080*oldZoomMul)); // TODO: Native reso should be constant somewhere
    const int gap = (int)(20*oldZoomMul);
    const V2i margin = v2i((int)(2000*oldZoomMul), (int)(2000*oldZoomMul));

    // Zoom around cursor
    if (zoomChanged)
    {
        V2f scaledCursorPos = v2i_to_v2f(gui.cursor_pos);// - v2f(screenW/2.f, screenH/2.f);
        V2f oldClientPos = v2i_to_v2f(gui_frame_scroll(&gui)) + scaledCursorPos;
        V2f worldPos = oldClientPos*(1.f/oldZoomMul);
        V2f newClientPos = worldPos*zoomMul;
        V2f newScreenPos = newClientPos - scaledCursorPos;

        gui_set_frame_scroll(&gui, v2f_to_v2i(newScreenPos));
    }

    for (auto x = 0u; x < mScreens.size(); x++)
    {
        for (auto y = 0u; y < mScreens[x].size(); y++)
        {
            GridScreen *screen = mScreens[x][y].get();
            if (!screen->hasTexture())
                continue;

            V2i pos = gui_turtle_pos(&gui) + v2i((camSize.x + gap)*x, (camSize.y + gap)*y) + margin;
            rend.drawQuad(screen->getTexHandle(mFs), 1.0f*pos.x, 1.0f*pos.y, 1.0f*camSize.x, 1.0f*camSize.y);
            gui_enlarge_bounding(&gui, pos + camSize + margin*2);
        }
    }
    rend.endLayer();
    gui_end_frame(&gui);
#endif
}

<commit_msg>Fixed background flickering when scrolling with mousewheel<commit_after>#include "gridmap.hpp"
#include "gui.hpp"
#include "oddlib/lvlarchive.hpp"
#include "renderer.hpp"
#include "oddlib/path.hpp"
#include "oddlib/bits_factory.hpp"
#include "logger.hpp"
#include <cassert>
#include "sdl_raii.hpp"
#include <algorithm> // min/max
#include <cmath>

Level::Level(GameData& gameData, IAudioController& /*audioController*/, FileSystem& fs)
    : mGameData(gameData), mFs(fs)
{

}

void Level::Update()
{
    if (mMap)
    {
        mMap->Update();
    }
}

void Level::Render(Renderer& rend, GuiContext& gui, int screenW, int screenH)
{
    RenderDebugPathSelection(rend, gui);
    if (mMap)
    {
        mMap->Render(rend, gui, screenW, screenH);
    }
}

void Level::RenderDebugPathSelection(Renderer& rend, GuiContext& gui)
{
    gui.next_window_pos = v2i(10, 300);
    gui_begin_window(&gui, "Paths", v2i(150, 400));

    static std::vector<std::pair<std::string, const GameData::PathEntry*>> items;
    if (items.empty())
    {

        for (const auto& pathGroup : mGameData.Paths())
        {
            for (const auto& path : pathGroup.second)
            {
                items.push_back(std::make_pair(pathGroup.first + " " + std::to_string(path.mPathChunkId), &path));
            }
        }
    }

    for (const auto& item : items)
    {
        if (gui_button(&gui, item.first.c_str()))
        {
            const GameData::PathEntry* entry = item.second;
            const std::string baseLvlNameUpper = item.first.substr(0, 2); // R1, MI etc
            std::string baseLvlNameLower = baseLvlNameUpper;

            // Open the LVL
            std::transform(baseLvlNameLower.begin(), baseLvlNameLower.end(), baseLvlNameLower.begin(), string_util::c_tolower);
            const std::string lvlName = baseLvlNameLower + ".lvl";

            LOG_INFO("Looking for LVL " << lvlName);

            auto chunkStream = mFs.ResourcePaths().OpenLvlFileChunkById(lvlName, baseLvlNameUpper + "PATH.BND", entry->mPathChunkId);
            if (chunkStream)
            {
                Oddlib::Path path(*chunkStream,
                    entry->mNumberOfCollisionItems,
                    entry->mObjectIndexTableOffset,
                    entry->mObjectDataOffset,
                    entry->mMapXSize,
                    entry->mMapYSize);
                mMap = std::make_unique<GridMap>(lvlName, path, mFs, rend);
            }
            else
            {
                LOG_ERROR("LVL or file in LVL not found");
            }
        }
    }

    gui_end_window(&gui);
}

GridScreen::GridScreen(const std::string& lvlName, const std::string& fileName, Renderer& rend)
    : mLvlName(lvlName),
      mFileName(fileName)
      , mTexHandle(0)
      , mRend(rend)
{

}

GridScreen::~GridScreen()
{
    mRend.destroyTexture(mTexHandle);
}

int GridScreen::getTexHandle(FileSystem& fs)
{
    if (!mTexHandle)
    {
        auto stream = fs.ResourcePaths().OpenLvlFileChunkByType(mLvlName, mFileName, Oddlib::MakeType('B', 'i', 't', 's'));
        if (stream)
        {
            auto bits = Oddlib::MakeBits(*stream);

            SDL_Surface* surf = bits->GetSurface();
            SDL_SurfacePtr converted;
            if (surf->format->format != SDL_PIXELFORMAT_RGB24)
            {
                converted.reset(SDL_ConvertSurfaceFormat(surf, SDL_PIXELFORMAT_RGB24, 0));
                surf = converted.get();
            }

#if 0
            SDL_SurfacePtr scaled;
            { // Scale the surface
                assert(surf->w == 640);
                assert(surf->h == 240);

                scaled.reset(SDL_CreateRGBSurface(0, 640, 480, 24, 0xFF, 0x00FF, 0x0000FF, 0));
                uint8_t *src = static_cast<uint8_t*>(surf->pixels);
                uint8_t *dst = static_cast<uint8_t*>(scaled->pixels);
                for (int y = 0; y < scaled->h; ++y) {
                    const int src_y = min(y, surf->h);
                    for (int x = 0; x < scaled->w; ++x) {
                        const int src_x = min(x, surf->w);

                        const int dst_ix = x*3 + scaled->pitch*y;
                        const int src_ix = src_x*3 + surf->pitch*src_y;

                        dst[dst_ix + 0] = src[src_ix + 0];
                        dst[dst_ix + 1] = src[src_ix + 1];
                        dst[dst_ix + 2] = src[src_ix + 2];
                    }
                }
            }
            mTexHandle = mRend.createTexture(GL_RGB, scaled->w, scaled->h, GL_RGB, GL_UNSIGNED_BYTE, scaled->pixels, false);
#else
            mTexHandle = mRend.createTexture(GL_RGB, surf->w, surf->h, GL_RGB, GL_UNSIGNED_BYTE, surf->pixels, false);
#endif
        }
    }
    return mTexHandle;
}

bool GridScreen::hasTexture() const
{
    bool onlySpaces = true;
    for (size_t i = 0; i < mFileName.size(); ++i) {
        if (mFileName[i] != ' ' && mFileName[i] != '\0')
        {
            onlySpaces = false;
            break;
        }
    }
    return !onlySpaces;
}

GridMap::GridMap(const std::string& lvlName, Oddlib::Path& path, FileSystem& fs, Renderer& rend)
    : mFs(fs), mLvlName(lvlName)
{
    mScreens.resize(path.XSize());
    for (auto& col : mScreens)
    {
        col.resize(path.YSize());
    }

    for (Uint32 x = 0; x < path.XSize(); x++)
    {
        for (Uint32 y = 0; y < path.YSize(); y++)
        {
            mScreens[x][y] = std::make_unique<GridScreen>(mLvlName, path.CameraFileName(x,y), rend);
        }
    }
}

void GridMap::Update()
{

}

void GridMap::Render(Renderer& rend, GuiContext& gui, int screenW, int screenH)
{
#if 0 // List of cams
    gui.next_window_pos = v2i(950, 50);
    gui_begin_window(&gui, "GridMap", v2i(200, 500));
    for (auto x = 0u; x < mScreens.size(); x++)
    {
        for (auto y = 0u; y < mScreens[0].size(); y++)
        {
            if (gui_button(&gui, gui_str(&gui, "cam_%i_%i|%s", (int)x, (int)y, mScreens[x][y]->FileName().c_str())))
            {
                mEditorScreenX = (int)x;
                mEditorScreenY = (int)y;
            }
        }
    }
    gui_end_window(&gui);

    if (mEditorScreenX >= static_cast<int>(mScreens.size()) || // Invalid x check
        (mEditorScreenX >= 0 && mEditorScreenY >= static_cast<int>(mScreens[mEditorScreenX].size()))) // Invalid y check
    {
        mEditorScreenX = mEditorScreenY = -1;
    }

    if (mEditorScreenX >= 0 && mEditorScreenY >= 0)
    {
        GridScreen *screen = mScreens[mEditorScreenX][mEditorScreenY].get();

        gui.next_window_pos = v2i(100, 100);

        V2i size = v2i(640, 480);
        gui_begin_window(&gui, "CAM", size);
        size = gui_window_client_size(&gui);

        rend.beginLayer(gui_layer(&gui));
        V2i pos = gui_turtle_pos(&gui);
        rend.drawQuad(screen->getTexHandle(mFs), 1.0f*pos.x, 1.0f*pos.y, 1.0f*size.x, 1.0f*size.y);
        rend.endLayer();

        gui_end_window(&gui);
    }
#else // Proper editor gui

    gui_begin_frame(&gui, "camArea", v2i(0, 0), v2i(screenW, screenH));
    rend.beginLayer(gui_layer(&gui));

    const float zoomBase = 1.2f;
    const float oldZoomMul = std::pow(zoomBase, 1.f*mZoomLevel);
    bool zoomChanged = false;
    if (gui.key_state[GUI_KEY_LCTRL] & GUI_KEYSTATE_DOWN_BIT)
    {
        mZoomLevel += gui.mouse_scroll;
        zoomChanged = (gui.mouse_scroll != 0);
    }
    const float zoomMul = std::pow(zoomBase, 1.f*mZoomLevel);
    // Use oldZoom because gui_set_frame_scroll below doesn't change scrolling in current frame. Could be changed though.
    const V2i camSize = v2i((int)(1440*oldZoomMul), (int)(1080*oldZoomMul)); // TODO: Native reso should be constant somewhere
    const int gap = (int)(20*oldZoomMul);
    const V2i margin = v2i((int)(2000*oldZoomMul), (int)(2000*oldZoomMul));

    // Zoom around cursor
    if (zoomChanged)
    {
        V2f scaledCursorPos = v2i_to_v2f(gui.cursor_pos);// - v2f(screenW/2.f, screenH/2.f);
        V2f oldClientPos = v2i_to_v2f(gui_frame_scroll(&gui)) + scaledCursorPos;
        V2f worldPos = oldClientPos*(1.f/oldZoomMul);
        V2f newClientPos = worldPos*zoomMul;
        V2f newScreenPos = newClientPos - scaledCursorPos;

        gui_set_frame_scroll(&gui, v2f_to_v2i(newScreenPos));
    }

    for (auto x = 0u; x < mScreens.size(); x++)
    {
        for (auto y = 0u; y < mScreens[x].size(); y++)
        {
            GridScreen *screen = mScreens[x][y].get();
            if (!screen->hasTexture())
                continue;

            V2i pos = gui_turtle_pos(&gui) + v2i((camSize.x + gap)*x, (camSize.y + gap)*y) + margin;
            rend.drawQuad(screen->getTexHandle(mFs), 1.0f*pos.x, 1.0f*pos.y, 1.0f*camSize.x, 1.0f*camSize.y);
            gui_enlarge_bounding(&gui, pos + camSize + margin*2);
        }
    }
    rend.endLayer();
    gui_end_frame(&gui);
#endif
}

<|endoftext|>


<commit_before>#include <Poco/Clock.h>
#include <Poco/Exception.h>
#include <Poco/DateTimeFormatter.h>
#include <Poco/Logger.h>

#include "di/Injectable.h"
#include "gws/GatewayScanController.h"
#include "util/LambdaTimerTask.h"

BEEEON_OBJECT_BEGIN(BeeeOn, GatewayScanController)
BEEEON_OBJECT_CASTABLE(StoppableLoop)
BEEEON_OBJECT_PROPERTY("gatewayRPC", &GatewayScanController::setGatewayRPC)
BEEEON_OBJECT_PROPERTY("defaultDuration", &GatewayScanController::setDefaultDuration)
BEEEON_OBJECT_PROPERTY("cleanupInterval", &GatewayScanController::setCleanupInterval)
BEEEON_OBJECT_PROPERTY("cleanupOlderThan", &GatewayScanController::setCleanupOlderThan)
BEEEON_OBJECT_END(BeeeOn, GatewayScanController)

using namespace std;
using namespace Poco;
using namespace BeeeOn;

const static Timespan MAX_SCAN_DURATION = 10 * Timespan::MINUTES;

GatewayScanController::ScanHandler::ScanHandler(const GatewayID &id):
	m_id(id)
{
}

const GatewayScan &GatewayScanController::ScanHandler::scan() const
{
	ScopedLock guard(const_cast<ScanHandler&>(*this));
	return m_scan;
}

GatewayScan &GatewayScanController::ScanHandler::scan()
{
	ScopedLock guard(const_cast<ScanHandler&>(*this));
	return m_scan;
}

void GatewayScanController::ScanHandler::onAny(GatewayRPCResult::Ptr r)
{
	Logger &logger = Loggable::forClass(typeid(GatewayScanController));
	ScanHandler::ScopedLock guard(*this);

	switch (r->status()) {
	case GatewayRPCResult::Status::PENDING:
	case GatewayRPCResult::Status::ACCEPTED:
		logger.information("scanning " + m_id.toString() + " is starting",
				__FILE__, __LINE__);
		m_scan.changeState(GatewayScan::SCAN_PROCESSING);
		break;

	case GatewayRPCResult::Status::SUCCESS:
		logger.information("scanning " + m_id.toString() + " in progress",
				__FILE__, __LINE__);
		m_scan.changeState(GatewayScan::SCAN_PROCESSING);
		break;

	case GatewayRPCResult::Status::FAILED:
		logger.error("scanning " + m_id.toString() + " has failed",
				__FILE__, __LINE__);

		m_scan.changeState(GatewayScan::SCAN_FAILED);
		break;

	case GatewayRPCResult::Status::TIMEOUT:
		logger.error("scanning " + m_id.toString() + " did not start on time",
				__FILE__, __LINE__);

		m_scan.changeState(GatewayScan::SCAN_FAILED);
		break;

	case GatewayRPCResult::Status::NOT_CONNECTED:
		logger.error("scanning " + m_id.toString() + " failed, not connected",
				__FILE__, __LINE__);

		m_scan.changeState(GatewayScan::SCAN_FAILED);
		break;
	}
}

GatewayScanController::CleanUpTask::CleanUpTask(
		FastMutex &lock,
		GatewayScanController::ScanMap &scanMap):
	m_olderThen(30 * Timespan::MINUTES),
	m_lock(lock),
	m_scanMap(scanMap)
{
}

void GatewayScanController::CleanUpTask::setOlderThan(
		const Poco::Timespan &olderThen)
{
	if (olderThen < 0)
		throw InvalidArgumentException("cleanup older-than timeout must be positive");

	m_olderThen = olderThen;
}

void GatewayScanController::CleanUpTask::run()
{
	FastMutex::ScopedLock guard(m_lock);
	const Clock started;
	const size_t total = m_scanMap.size();
	size_t count = 0;

	logger().information("cleaning up scan records ("
			+ to_string(total) + ")",
			__FILE__, __LINE__);

	for (auto it = m_scanMap.begin(); it != m_scanMap.end();) {
		ScanHandler::Ptr context = it->second;
		ScanHandler::ScopedLock guard(*context);

		if (logger().debug()) {
			logger().debug("cleaning up scan record for "
					+ it->first.toString(),
					__FILE__, __LINE__);
		}

		const auto &started = context->scan().started();
		const auto &duration = context->scan().duration();
		const Timestamp now;

		switch (context->scan().state()) {
		case GatewayScan::SCAN_IDLE:
			it = m_scanMap.erase(it);
			count += 1;
			break;

		case GatewayScan::SCAN_FINISHED:
		case GatewayScan::SCAN_FAILED:
			if (started + duration < now - m_olderThen) {
				it = m_scanMap.erase(it);
				count += 1;
			}
			else {
				++it;
			}

			break;

		case GatewayScan::SCAN_WAITING:
		case GatewayScan::SCAN_PROCESSING:
			if (started + duration < now) {
				logger().warning("scan record for " + it->first.toString()
						+ " was not stopped on time, marking as failed",
						__FILE__, __LINE__);
				context->scan().changeState(GatewayScan::SCAN_FAILED);
			}

			++it;
			break;

		default:
			++it;
		}
	}

	logger().notice("cleaned up "
			+ to_string(count) + "/" + to_string(total)
			+ " scan records in "
			+ to_string(started.elapsed()) + "us",
			__FILE__, __LINE__);
}

GatewayScanController::GatewayScanController():
	m_defaultDuration(2 * Timespan::MINUTES),
	m_cleanupInterval(15 * Timespan::MINUTES),
	m_cleanupTask(new CleanUpTask(m_lock, m_scanMap))
{
}

void GatewayScanController::setDefaultDuration(const Poco::Timespan &duration)
{
	if (duration.totalSeconds() <= 0)
		throw InvalidArgumentException("duration must be at least a second");

	if (duration > MAX_SCAN_DURATION)
		throw InvalidArgumentException("too long default duration");

	m_defaultDuration = duration;
}

void GatewayScanController::setCleanupInterval(const Timespan &interval)
{
	if (interval.totalMilliseconds() <= 0)
		throw InvalidArgumentException("cleanup interval must be at least a millisecond");

	m_cleanupInterval = interval;
}

void GatewayScanController::setCleanupOlderThan(const Timespan &time)
{
	m_cleanupTask->setOlderThan(time);
}

void GatewayScanController::setGatewayRPC(GatewayRPC::Ptr rpc)
{
	m_rpc = rpc;
}

void GatewayScanController::start()
{
	logger().information("starting gateway scan controller");

	m_timer.scheduleAtFixedRate(
		m_cleanupTask,
		m_cleanupInterval.totalMilliseconds(),
		m_cleanupInterval.totalMilliseconds());
}

void GatewayScanController::stop()
{
	logger().information("stopping gateway scan controller");

	m_timer.cancel(true);

	FastMutex::ScopedLock guard(m_lock);
	m_scanMap.clear();
}

GatewayScan GatewayScanController::scan(const Gateway &gateway, const Timespan &timeout)
{
	ScopedLockWithUnlock<FastMutex> guard(m_lock);

	auto it = m_scanMap.find(gateway.id());
	if (it != m_scanMap.end())
		return it->second->scan();

	const auto id = gateway.id();
	const Timespan &duration = timeout <= 0? m_defaultDuration : timeout;

	if (duration > MAX_SCAN_DURATION)
		throw InvalidArgumentException("too big scanning timeout given");

	ScanHandler::Ptr context = new ScanHandler(id);
	context->scan().setDuration(duration);

	m_scanMap.emplace(id, context);

	guard.unlock();

	const Clock start;

	m_rpc->sendListen(
		context,
		gateway,
		duration
	);

	m_timer.schedule(new LambdaTimerTask([start, id, context]() mutable
		{
			Logger &logger = Loggable::forClass(typeid(GatewayScanController));

			ScanHandler::ScopedLock guard(*context);
			GatewayScan &scan = context->scan();

			const Timespan &duration = scan.started().elapsed();

			switch (scan.state()) {
			case GatewayScan::SCAN_FINISHED:
				logger.warning("scanning of "
						+ id.toString()
						+ " is already finished",
						__FILE__, __LINE__);
				break;

			case GatewayScan::SCAN_WAITING:
			case GatewayScan::SCAN_PROCESSING:
				logger.information("scanning of "
						+ id.toString()
						+ " was finished after "
						+ DateTimeFormatter::format(duration),
						__FILE__, __LINE__);

				scan.changeState(GatewayScan::SCAN_FINISHED);
				break;

			default:
				if (logger.debug()) {
					logger.debug("skipping explicit finish of "
							+ id.toString() + " scan",
							__FILE__, __LINE__);
				}
				break;
			}
		}),
		start + duration.totalMicroseconds());

	return context->scan();
}

GatewayScan GatewayScanController::find(const Gateway &gateway)
{
	FastMutex::ScopedLock guard(m_lock);

	auto it = m_scanMap.find(gateway.id());
	if (it != m_scanMap.end())
		return it->second->scan();

	return GatewayScan::createIdle();
}
<commit_msg>GatewayScanController::ScanHandler: distuingish between pending and accept<commit_after>#include <Poco/Clock.h>
#include <Poco/Exception.h>
#include <Poco/DateTimeFormatter.h>
#include <Poco/Logger.h>

#include "di/Injectable.h"
#include "gws/GatewayScanController.h"
#include "util/LambdaTimerTask.h"

BEEEON_OBJECT_BEGIN(BeeeOn, GatewayScanController)
BEEEON_OBJECT_CASTABLE(StoppableLoop)
BEEEON_OBJECT_PROPERTY("gatewayRPC", &GatewayScanController::setGatewayRPC)
BEEEON_OBJECT_PROPERTY("defaultDuration", &GatewayScanController::setDefaultDuration)
BEEEON_OBJECT_PROPERTY("cleanupInterval", &GatewayScanController::setCleanupInterval)
BEEEON_OBJECT_PROPERTY("cleanupOlderThan", &GatewayScanController::setCleanupOlderThan)
BEEEON_OBJECT_END(BeeeOn, GatewayScanController)

using namespace std;
using namespace Poco;
using namespace BeeeOn;

const static Timespan MAX_SCAN_DURATION = 10 * Timespan::MINUTES;

GatewayScanController::ScanHandler::ScanHandler(const GatewayID &id):
	m_id(id)
{
}

const GatewayScan &GatewayScanController::ScanHandler::scan() const
{
	ScopedLock guard(const_cast<ScanHandler&>(*this));
	return m_scan;
}

GatewayScan &GatewayScanController::ScanHandler::scan()
{
	ScopedLock guard(const_cast<ScanHandler&>(*this));
	return m_scan;
}

void GatewayScanController::ScanHandler::onAny(GatewayRPCResult::Ptr r)
{
	Logger &logger = Loggable::forClass(typeid(GatewayScanController));
	ScanHandler::ScopedLock guard(*this);

	switch (r->status()) {
	case GatewayRPCResult::Status::PENDING:
		logger.information("scanning " + m_id.toString() + " requested",
				__FILE__, __LINE__);
		m_scan.changeState(GatewayScan::SCAN_PROCESSING);
		break;

	case GatewayRPCResult::Status::ACCEPTED:
		logger.information("scanning " + m_id.toString() + " is starting",
				__FILE__, __LINE__);
		m_scan.changeState(GatewayScan::SCAN_PROCESSING);
		break;

	case GatewayRPCResult::Status::SUCCESS:
		logger.information("scanning " + m_id.toString() + " in progress",
				__FILE__, __LINE__);
		m_scan.changeState(GatewayScan::SCAN_PROCESSING);
		break;

	case GatewayRPCResult::Status::FAILED:
		logger.error("scanning " + m_id.toString() + " has failed",
				__FILE__, __LINE__);

		m_scan.changeState(GatewayScan::SCAN_FAILED);
		break;

	case GatewayRPCResult::Status::TIMEOUT:
		logger.error("scanning " + m_id.toString() + " did not start on time",
				__FILE__, __LINE__);

		m_scan.changeState(GatewayScan::SCAN_FAILED);
		break;

	case GatewayRPCResult::Status::NOT_CONNECTED:
		logger.error("scanning " + m_id.toString() + " failed, not connected",
				__FILE__, __LINE__);

		m_scan.changeState(GatewayScan::SCAN_FAILED);
		break;
	}
}

GatewayScanController::CleanUpTask::CleanUpTask(
		FastMutex &lock,
		GatewayScanController::ScanMap &scanMap):
	m_olderThen(30 * Timespan::MINUTES),
	m_lock(lock),
	m_scanMap(scanMap)
{
}

void GatewayScanController::CleanUpTask::setOlderThan(
		const Poco::Timespan &olderThen)
{
	if (olderThen < 0)
		throw InvalidArgumentException("cleanup older-than timeout must be positive");

	m_olderThen = olderThen;
}

void GatewayScanController::CleanUpTask::run()
{
	FastMutex::ScopedLock guard(m_lock);
	const Clock started;
	const size_t total = m_scanMap.size();
	size_t count = 0;

	logger().information("cleaning up scan records ("
			+ to_string(total) + ")",
			__FILE__, __LINE__);

	for (auto it = m_scanMap.begin(); it != m_scanMap.end();) {
		ScanHandler::Ptr context = it->second;
		ScanHandler::ScopedLock guard(*context);

		if (logger().debug()) {
			logger().debug("cleaning up scan record for "
					+ it->first.toString(),
					__FILE__, __LINE__);
		}

		const auto &started = context->scan().started();
		const auto &duration = context->scan().duration();
		const Timestamp now;

		switch (context->scan().state()) {
		case GatewayScan::SCAN_IDLE:
			it = m_scanMap.erase(it);
			count += 1;
			break;

		case GatewayScan::SCAN_FINISHED:
		case GatewayScan::SCAN_FAILED:
			if (started + duration < now - m_olderThen) {
				it = m_scanMap.erase(it);
				count += 1;
			}
			else {
				++it;
			}

			break;

		case GatewayScan::SCAN_WAITING:
		case GatewayScan::SCAN_PROCESSING:
			if (started + duration < now) {
				logger().warning("scan record for " + it->first.toString()
						+ " was not stopped on time, marking as failed",
						__FILE__, __LINE__);
				context->scan().changeState(GatewayScan::SCAN_FAILED);
			}

			++it;
			break;

		default:
			++it;
		}
	}

	logger().notice("cleaned up "
			+ to_string(count) + "/" + to_string(total)
			+ " scan records in "
			+ to_string(started.elapsed()) + "us",
			__FILE__, __LINE__);
}

GatewayScanController::GatewayScanController():
	m_defaultDuration(2 * Timespan::MINUTES),
	m_cleanupInterval(15 * Timespan::MINUTES),
	m_cleanupTask(new CleanUpTask(m_lock, m_scanMap))
{
}

void GatewayScanController::setDefaultDuration(const Poco::Timespan &duration)
{
	if (duration.totalSeconds() <= 0)
		throw InvalidArgumentException("duration must be at least a second");

	if (duration > MAX_SCAN_DURATION)
		throw InvalidArgumentException("too long default duration");

	m_defaultDuration = duration;
}

void GatewayScanController::setCleanupInterval(const Timespan &interval)
{
	if (interval.totalMilliseconds() <= 0)
		throw InvalidArgumentException("cleanup interval must be at least a millisecond");

	m_cleanupInterval = interval;
}

void GatewayScanController::setCleanupOlderThan(const Timespan &time)
{
	m_cleanupTask->setOlderThan(time);
}

void GatewayScanController::setGatewayRPC(GatewayRPC::Ptr rpc)
{
	m_rpc = rpc;
}

void GatewayScanController::start()
{
	logger().information("starting gateway scan controller");

	m_timer.scheduleAtFixedRate(
		m_cleanupTask,
		m_cleanupInterval.totalMilliseconds(),
		m_cleanupInterval.totalMilliseconds());
}

void GatewayScanController::stop()
{
	logger().information("stopping gateway scan controller");

	m_timer.cancel(true);

	FastMutex::ScopedLock guard(m_lock);
	m_scanMap.clear();
}

GatewayScan GatewayScanController::scan(const Gateway &gateway, const Timespan &timeout)
{
	ScopedLockWithUnlock<FastMutex> guard(m_lock);

	auto it = m_scanMap.find(gateway.id());
	if (it != m_scanMap.end())
		return it->second->scan();

	const auto id = gateway.id();
	const Timespan &duration = timeout <= 0? m_defaultDuration : timeout;

	if (duration > MAX_SCAN_DURATION)
		throw InvalidArgumentException("too big scanning timeout given");

	ScanHandler::Ptr context = new ScanHandler(id);
	context->scan().setDuration(duration);

	m_scanMap.emplace(id, context);

	guard.unlock();

	const Clock start;

	m_rpc->sendListen(
		context,
		gateway,
		duration
	);

	m_timer.schedule(new LambdaTimerTask([start, id, context]() mutable
		{
			Logger &logger = Loggable::forClass(typeid(GatewayScanController));

			ScanHandler::ScopedLock guard(*context);
			GatewayScan &scan = context->scan();

			const Timespan &duration = scan.started().elapsed();

			switch (scan.state()) {
			case GatewayScan::SCAN_FINISHED:
				logger.warning("scanning of "
						+ id.toString()
						+ " is already finished",
						__FILE__, __LINE__);
				break;

			case GatewayScan::SCAN_WAITING:
			case GatewayScan::SCAN_PROCESSING:
				logger.information("scanning of "
						+ id.toString()
						+ " was finished after "
						+ DateTimeFormatter::format(duration),
						__FILE__, __LINE__);

				scan.changeState(GatewayScan::SCAN_FINISHED);
				break;

			default:
				if (logger.debug()) {
					logger.debug("skipping explicit finish of "
							+ id.toString() + " scan",
							__FILE__, __LINE__);
				}
				break;
			}
		}),
		start + duration.totalMicroseconds());

	return context->scan();
}

GatewayScan GatewayScanController::find(const Gateway &gateway)
{
	FastMutex::ScopedLock guard(m_lock);

	auto it = m_scanMap.find(gateway.id());
	if (it != m_scanMap.end())
		return it->second->scan();

	return GatewayScan::createIdle();
}
<|endoftext|>


<commit_before>/** \file    add_author_synonyms.cc
 *  \brief   Adds author synonyms to each record.
 *  \author  Oliver Obenland
 */

/*
    Copyright (C) 2016-2017, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <map>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

static unsigned modified_count(0);
static unsigned record_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << " master_marc_input norm_data_marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

void RemoveCommasDuplicatesAndEmptyEntries(std::vector<std::string> * const vector) {
    std::vector<std::string> cleaned_up_vector;
    std::set<std::string> unique_entries;

    for (auto &entry : *vector) {
        StringUtil::RemoveChars(",", &entry);

        if (entry.empty())
            continue;

        const bool is_new_entry(unique_entries.emplace(entry).second);
        if (is_new_entry)
            cleaned_up_vector.emplace_back(std::move(entry));
    }
    vector->swap(cleaned_up_vector);
}

std::string ExtractNameFromSubfields(const std::string &field_contents, const std::string &subfield_codes) {
    const Subfields subfields(field_contents);
    std::vector<std::string> subfield_values;
    if (subfields.extractSubfields(subfield_codes, &subfield_values) == 0)
        return "";

    std::sort(subfield_values.begin(), subfield_values.end());
    return StringUtil::Join(subfield_values, ' ');
}

void ExtractSynonyms(MarcReader * const marc_reader, std::map<std::string, std::string> &author_to_synonyms_map,
                     const std::string &field_list)
{
    std::set<std::string> synonyms;
    std::vector<std::string> tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(field_list, ':', &tags_and_subfield_codes) < 2))
        logger->error("in ExtractSynonymsAndWriteSynonymMap: need at least two fields!");
    unsigned count(0);
    while (const MarcRecord record = marc_reader->read()) {
        ++count;

        const size_t primary_name_field_index(record.getFieldIndex(tags_and_subfield_codes[0].substr(0, 3)));
        if (primary_name_field_index == MarcRecord::FIELD_NOT_FOUND)
            continue;

        const std::string primary_name(ExtractNameFromSubfields(record.getFieldData(primary_name_field_index),
                                                                tags_and_subfield_codes[0].substr(3)));
        if (unlikely(primary_name.empty()))
            continue;

        std::vector<std::string> alternatives;
        alternatives.emplace_back(primary_name);
        if (author_to_synonyms_map.find(primary_name) != author_to_synonyms_map.end())
            continue;

        for (unsigned i(1); i < tags_and_subfield_codes.size(); ++i) {
            const std::string tag(tags_and_subfield_codes[i].substr(0, 3));
            const std::string secondary_field_subfield_codes(tags_and_subfield_codes[i].substr(3));
            for (size_t secondary_name_field_index(record.getFieldIndex(tag));
                 secondary_name_field_index < record.getNumberOfFields() and record.getTag(secondary_name_field_index) == tag;
                 ++secondary_name_field_index)
            {
                const std::string secondary_name(ExtractNameFromSubfields(record.getFieldData(secondary_name_field_index), secondary_field_subfield_codes));
                if (not secondary_name.empty())
                    alternatives.emplace_back(secondary_name);
            }
        }
        RemoveCommasDuplicatesAndEmptyEntries(&alternatives);
        if (alternatives.size() <= 1)
            continue;

        alternatives.erase(alternatives.begin());
        author_to_synonyms_map.emplace(primary_name, StringUtil::Join(alternatives, ','));
    }

    std::cout << "Found synonyms for " << author_to_synonyms_map.size() << " authors while processing " << count
              << " norm data records.\n";
}

const std::string SYNOMYM_FIELD("101"); // This must be an o/w unused field!

void ProcessRecord(MarcRecord * const record, const std::map<std::string, std::string> &author_to_synonyms_map,
                   const std::string &primary_author_field)
{
    if (unlikely(record->getFieldIndex(SYNOMYM_FIELD) != MarcRecord::FIELD_NOT_FOUND))
        logger->error("field " + SYNOMYM_FIELD + " is apparently already in use in at least some title records!");

    const size_t primary_name_field_index(record->getFieldIndex(primary_author_field.substr(0, 3)));
    if (primary_name_field_index == MarcRecord::FIELD_NOT_FOUND)
        return;

    const std::string primary_name(ExtractNameFromSubfields(record->getFieldData(primary_name_field_index),
                                                            primary_author_field.substr(3)));
    if (unlikely(primary_name.empty()))
        return;

    const auto synonyms_iterator = author_to_synonyms_map.find(primary_name);
    if (synonyms_iterator == author_to_synonyms_map.end())
        return;

    const std::string synonyms = synonyms_iterator->second;
    Subfields subfields(/* indicator1 = */' ', /* indicator2 = */' ');
    subfields.addSubfield('a', synonyms);

    if (not record->insertField(SYNOMYM_FIELD, subfields)) {
        logger->warning("Not enough room to add a " + SYNOMYM_FIELD + " field! (Control number: "
                        + record->getControlNumber() + ")");
        return;
    }
    ++modified_count;
}

void AddAuthorSynonyms(MarcReader * const marc_reader, MarcWriter * marc_writer,
                       const std::map<std::string, std::string> &author_to_synonyms_map,
                       const std::string &primary_author_field)
{
    while (MarcRecord record = marc_reader->read()) {
        ProcessRecord(&record, author_to_synonyms_map, primary_author_field);
        marc_writer->write(record);
        ++record_count;
    }

    std::cerr << "Modified " << modified_count << " of " << record_count << " record(s).\n";
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 4)
        Usage();

    const std::string marc_input_filename(argv[1]);
    const std::string authority_data_marc_input_filename(argv[2]);
    const std::string marc_output_filename(argv[3]);
    if (unlikely(marc_input_filename == marc_output_filename))
        logger->error("Title input file name equals title output file name!");
    if (unlikely(authority_data_marc_input_filename == marc_output_filename))
        logger->error("Authority data input file name equals MARC output file name!");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(marc_input_filename, MarcReader::BINARY));
    std::unique_ptr<MarcReader> authority_reader(MarcReader::Factory(authority_data_marc_input_filename,
                                                                     MarcReader::BINARY));
    std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename, MarcWriter::BINARY));

    try {
        std::map<std::string, std::string> author_to_synonyms_map;
        ExtractSynonyms(authority_reader.get(), author_to_synonyms_map, "100abcd:400abcd");
        AddAuthorSynonyms(marc_reader.get(), marc_writer.get(), author_to_synonyms_map, "100abcd");
    } catch (const std::exception &x) {
        logger->error("caught exception: " + std::string(x.what()));
    }
}
<commit_msg>Changed field 101 to 109 because of non-standard, local-only use.<commit_after>/** \file    add_author_synonyms.cc
 *  \brief   Adds author synonyms to each record.
 *  \author  Oliver Obenland
 */

/*
    Copyright (C) 2016-2017, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <map>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "MarcWriter.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

static unsigned modified_count(0);
static unsigned record_count(0);

void Usage() {
    std::cerr << "Usage: " << ::progname << " master_marc_input norm_data_marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

void RemoveCommasDuplicatesAndEmptyEntries(std::vector<std::string> * const vector) {
    std::vector<std::string> cleaned_up_vector;
    std::set<std::string> unique_entries;

    for (auto &entry : *vector) {
        StringUtil::RemoveChars(",", &entry);

        if (entry.empty())
            continue;

        const bool is_new_entry(unique_entries.emplace(entry).second);
        if (is_new_entry)
            cleaned_up_vector.emplace_back(std::move(entry));
    }
    vector->swap(cleaned_up_vector);
}

std::string ExtractNameFromSubfields(const std::string &field_contents, const std::string &subfield_codes) {
    const Subfields subfields(field_contents);
    std::vector<std::string> subfield_values;
    if (subfields.extractSubfields(subfield_codes, &subfield_values) == 0)
        return "";

    std::sort(subfield_values.begin(), subfield_values.end());
    return StringUtil::Join(subfield_values, ' ');
}

void ExtractSynonyms(MarcReader * const marc_reader, std::map<std::string, std::string> &author_to_synonyms_map,
                     const std::string &field_list)
{
    std::set<std::string> synonyms;
    std::vector<std::string> tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(field_list, ':', &tags_and_subfield_codes) < 2))
        logger->error("in ExtractSynonymsAndWriteSynonymMap: need at least two fields!");
    unsigned count(0);
    while (const MarcRecord record = marc_reader->read()) {
        ++count;

        const size_t primary_name_field_index(record.getFieldIndex(tags_and_subfield_codes[0].substr(0, 3)));
        if (primary_name_field_index == MarcRecord::FIELD_NOT_FOUND)
            continue;

        const std::string primary_name(ExtractNameFromSubfields(record.getFieldData(primary_name_field_index),
                                                                tags_and_subfield_codes[0].substr(3)));
        if (unlikely(primary_name.empty()))
            continue;

        std::vector<std::string> alternatives;
        alternatives.emplace_back(primary_name);
        if (author_to_synonyms_map.find(primary_name) != author_to_synonyms_map.end())
            continue;

        for (unsigned i(1); i < tags_and_subfield_codes.size(); ++i) {
            const std::string tag(tags_and_subfield_codes[i].substr(0, 3));
            const std::string secondary_field_subfield_codes(tags_and_subfield_codes[i].substr(3));
            for (size_t secondary_name_field_index(record.getFieldIndex(tag));
                 secondary_name_field_index < record.getNumberOfFields() and record.getTag(secondary_name_field_index) == tag;
                 ++secondary_name_field_index)
            {
                const std::string secondary_name(ExtractNameFromSubfields(record.getFieldData(secondary_name_field_index), secondary_field_subfield_codes));
                if (not secondary_name.empty())
                    alternatives.emplace_back(secondary_name);
            }
        }
        RemoveCommasDuplicatesAndEmptyEntries(&alternatives);
        if (alternatives.size() <= 1)
            continue;

        alternatives.erase(alternatives.begin());
        author_to_synonyms_map.emplace(primary_name, StringUtil::Join(alternatives, ','));
    }

    std::cout << "Found synonyms for " << author_to_synonyms_map.size() << " authors while processing " << count
              << " norm data records.\n";
}

const std::string SYNOMYM_FIELD("109"); // This must be an o/w unused field!

void ProcessRecord(MarcRecord * const record, const std::map<std::string, std::string> &author_to_synonyms_map,
                   const std::string &primary_author_field)
{
    if (unlikely(record->getFieldIndex(SYNOMYM_FIELD) != MarcRecord::FIELD_NOT_FOUND))
        logger->error("field " + SYNOMYM_FIELD + " is apparently already in use in at least some title records!");

    const size_t primary_name_field_index(record->getFieldIndex(primary_author_field.substr(0, 3)));
    if (primary_name_field_index == MarcRecord::FIELD_NOT_FOUND)
        return;

    const std::string primary_name(ExtractNameFromSubfields(record->getFieldData(primary_name_field_index),
                                                            primary_author_field.substr(3)));
    if (unlikely(primary_name.empty()))
        return;

    const auto synonyms_iterator = author_to_synonyms_map.find(primary_name);
    if (synonyms_iterator == author_to_synonyms_map.end())
        return;

    const std::string synonyms = synonyms_iterator->second;
    Subfields subfields(/* indicator1 = */' ', /* indicator2 = */' ');
    subfields.addSubfield('a', synonyms);

    if (not record->insertField(SYNOMYM_FIELD, subfields)) {
        logger->warning("Not enough room to add a " + SYNOMYM_FIELD + " field! (Control number: "
                        + record->getControlNumber() + ")");
        return;
    }
    ++modified_count;
}

void AddAuthorSynonyms(MarcReader * const marc_reader, MarcWriter * marc_writer,
                       const std::map<std::string, std::string> &author_to_synonyms_map,
                       const std::string &primary_author_field)
{
    while (MarcRecord record = marc_reader->read()) {
        ProcessRecord(&record, author_to_synonyms_map, primary_author_field);
        marc_writer->write(record);
        ++record_count;
    }

    std::cerr << "Modified " << modified_count << " of " << record_count << " record(s).\n";
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 4)
        Usage();

    const std::string marc_input_filename(argv[1]);
    const std::string authority_data_marc_input_filename(argv[2]);
    const std::string marc_output_filename(argv[3]);
    if (unlikely(marc_input_filename == marc_output_filename))
        logger->error("Title input file name equals title output file name!");
    if (unlikely(authority_data_marc_input_filename == marc_output_filename))
        logger->error("Authority data input file name equals MARC output file name!");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(marc_input_filename, MarcReader::BINARY));
    std::unique_ptr<MarcReader> authority_reader(MarcReader::Factory(authority_data_marc_input_filename,
                                                                     MarcReader::BINARY));
    std::unique_ptr<MarcWriter> marc_writer(MarcWriter::Factory(marc_output_filename, MarcWriter::BINARY));

    try {
        std::map<std::string, std::string> author_to_synonyms_map;
        ExtractSynonyms(authority_reader.get(), author_to_synonyms_map, "100abcd:400abcd");
        AddAuthorSynonyms(marc_reader.get(), marc_writer.get(), author_to_synonyms_map, "100abcd");
    } catch (const std::exception &x) {
        logger->error("caught exception: " + std::string(x.what()));
    }
}
<|endoftext|>