<commit_before>/******************************************************************************
 * Copyright 2018 The Apollo Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *****************************************************************************/

#include "modules/dreamview/backend/hmi/hmi_worker.h"

#include <vector>

#include "gflags/gflags.h"
#include "modules/common/adapters/adapter_manager.h"
#include "modules/common/kv_db/kv_db.h"
#include "modules/common/util/file.h"
#include "modules/common/util/map_util.h"
#include "modules/common/util/string_tokenizer.h"
#include "modules/common/util/string_util.h"
#include "modules/control/proto/pad_msg.pb.h"
#include "modules/data/util/info_collector.h"
#include "modules/dreamview/backend/common/dreamview_gflags.h"
#include "modules/dreamview/backend/hmi/vehicle_manager.h"

DEFINE_string(map_data_path, "/apollo/modules/map/data", "Path to map data.");

DEFINE_string(vehicle_data_path, "/apollo/modules/calibration/data",
              "Path to vehicle data.");

namespace apollo {
namespace dreamview {
namespace {

using apollo::canbus::Chassis;
using apollo::common::adapter::AdapterManager;
using apollo::common::util::ContainsKey;
using apollo::common::util::FindOrNull;
using apollo::common::util::StringTokenizer;
using apollo::control::DrivingAction;
using google::protobuf::Map;
using RLock = boost::shared_lock<boost::shared_mutex>;
using WLock = boost::unique_lock<boost::shared_mutex>;

constexpr char kNavigationModeName[] = "Navigation";

// Convert a string to be title-like. E.g.: "hello_world" -> "Hello World".
std::string TitleCase(const std::string &origin,
                      const std::string &delimiter = "_") {
  std::vector<std::string> parts = StringTokenizer::Split(origin, delimiter);
  for (auto &part : parts) {
    if (!part.empty()) {
      // Upper case the first char.
      part[0] = toupper(part[0]);
    }
  }

  return apollo::common::util::PrintIter(parts);
}

// List subdirs and return a dict of {subdir_title: subdir_path}.
Map<std::string, std::string> ListDirAsDict(const std::string &dir) {
  Map<std::string, std::string> result;
  const auto subdirs = apollo::common::util::ListSubDirectories(dir);
  for (const auto &subdir : subdirs) {
    const auto subdir_title = TitleCase(subdir);
    const auto subdir_path = apollo::common::util::StrCat(dir, "/", subdir);
    result.insert({subdir_title, subdir_path});
  }
  return result;
}

// Run a supported command of some component, return the ret code.
int RunComponentCommand(const Map<std::string, Component> &components,
                        const std::string &component_name,
                        const std::string &command_name) {
  const auto *component = FindOrNull(components, component_name);
  if (component == nullptr) {
    AERROR << "Cannot find component " << component_name;
    return -1;
  }
  const auto *cmd = FindOrNull(component->supported_commands(), command_name);
  if (cmd == nullptr) {
    AERROR << "Cannot find command " << component_name << "." << command_name;
    return -1;
  }
  AINFO << "Execute system command: " << *cmd;
  const int ret = std::system(cmd->c_str());

  AERROR_IF(ret != 0) << "Command returns " << ret << ": " << *cmd;
  return ret;
}

template <class FlagType, class ValueType>
void SetGlobalFlag(const std::string &flag_name, const ValueType &value,
                   FlagType *flag) {
  static constexpr char kGlobalFlagfile[] =
      "/apollo/modules/common/data/global_flagfile.txt";
  if (*flag != value) {
    *flag = value;
    // Overwrite global flagfile.
    std::ofstream fout(kGlobalFlagfile, std::ios_base::app);
    CHECK(fout) << "Fail to open global flagfile " << kGlobalFlagfile;
    fout << "\n--" << flag_name << "=" << value << std::endl;
  }
}

}  // namespace

HMIWorker::HMIWorker() {
  // Init HMIConfig.
  CHECK(common::util::GetProtoFromFile(FLAGS_hmi_config_filename, &config_))
      << "Unable to parse HMI config file " << FLAGS_hmi_config_filename;
  config_.set_docker_image(apollo::data::InfoCollector::GetDockerImage());

  // If the module path doesn't exist, remove it from list.
  auto *modules = config_.mutable_modules();
  for (auto iter = modules->begin(); iter != modules->end();) {
    const auto &conf = iter->second;
    if (conf.has_path() && !common::util::PathExists(conf.path())) {
      iter = modules->erase(iter);
    } else {
      ++iter;
    }
  }

  // Get available maps and vehicles by listing data directory.
  *config_.mutable_available_maps() = ListDirAsDict(FLAGS_map_data_path);
  *config_.mutable_available_vehicles() =
      ListDirAsDict(FLAGS_vehicle_data_path);
  config_.set_utm_zone_id(FLAGS_local_utm_zone_id);
  ADEBUG << "Loaded HMI config: " << config_.DebugString();

  // Init HMIStatus.
  const auto &modes = config_.modes();
  if (FLAGS_use_navigation_mode && ContainsKey(modes, kNavigationModeName)) {
    // If the FLAG_use_navigation_mode is set, set it in HMIStatus.
    status_.set_current_mode(kNavigationModeName);
  } else if (!ContainsKey(modes, status_.current_mode())) {
    CHECK(!modes.empty()) << "No available modes to run vehicle.";
    // If the default mode is unavailable, select the first one.
    status_.set_current_mode(modes.begin()->first);
  }

  // If the FLAGS_map_dir is set, set it in HMIStatus.
  if (!FLAGS_map_dir.empty()) {
    for (const auto &entry : config_.available_maps()) {
      // entry is (map_name, map_path)
      if (entry.second == FLAGS_map_dir) {
        status_.set_current_map(entry.first);
        break;
      }
    }
  }
}

bool HMIWorker::Trigger(const HMIAction action) {
  AINFO << "HMIAction " << HMIAction_Name(action) << " was triggered!";
  switch (action) {
    case HMIAction::SETUP:
      RunModeCommand("start");
      break;
    case HMIAction::AUTO_MODE:
      return ChangeToDrivingMode(Chassis::COMPLETE_AUTO_DRIVE);
    case HMIAction::DISENGAGE:
      return ChangeToDrivingMode(Chassis::COMPLETE_MANUAL);
    default:
      AERROR << "HMIAction not implemented, yet!";
      return false;
  }
  return true;
}

int HMIWorker::RunModuleCommand(const std::string &module,
                                const std::string &command) {
  return RunComponentCommand(config_.modules(), module, command);
}

int HMIWorker::RunHardwareCommand(const std::string &hardware,
                                  const std::string &command) {
  return RunComponentCommand(config_.hardware(), hardware, command);
}

int HMIWorker::RunToolCommand(const std::string &tool,
                              const std::string &command) {
  return RunComponentCommand(config_.tools(), tool, command);
}

void HMIWorker::SubmitDriveEvent(const uint64_t event_time_ms,
                                 const std::string &event_msg) {
  apollo::common::DriveEvent drive_event;
  AdapterManager::FillDriveEventHeader("HMI", &drive_event);
  drive_event.mutable_header()->set_timestamp_sec(event_time_ms / 1000.0);
  drive_event.set_event(event_msg);
  AdapterManager::PublishDriveEvent(drive_event);
}

bool HMIWorker::ChangeToDrivingMode(const Chassis::DrivingMode mode) {
  // Always reset to MANUAL mode before changing to other mode.
  if (mode != Chassis::COMPLETE_MANUAL) {
    if (!ChangeToDrivingMode(Chassis::COMPLETE_MANUAL)) {
      AERROR << "Failed to reset to MANUAL mode before changing to "
             << Chassis::DrivingMode_Name(mode);
      return false;
    }
  }

  control::PadMessage pad;
  switch (mode) {
    case Chassis::COMPLETE_MANUAL:
      pad.set_action(DrivingAction::RESET);
      break;
    case Chassis::COMPLETE_AUTO_DRIVE:
      pad.set_action(DrivingAction::START);
      break;
    default:
      AFATAL << "Unknown action to change driving mode to "
             << Chassis::DrivingMode_Name(mode);
  }

  constexpr int kMaxTries = 3;
  constexpr auto kTryInterval = std::chrono::milliseconds(500);
  auto *chassis = CHECK_NOTNULL(AdapterManager::GetChassis());
  for (int i = 0; i < kMaxTries; ++i) {
    // Send driving action periodically until entering target driving mode.
    AdapterManager::FillPadHeader("HMI", &pad);
    AdapterManager::PublishPad(pad);

    std::this_thread::sleep_for(kTryInterval);
    chassis->Observe();
    if (chassis->Empty()) {
      AERROR << "No Chassis message received!";
    } else if (chassis->GetLatestObserved().driving_mode() == mode) {
      return true;
    }
  }
  AERROR << "Failed to change driving mode to "
         << Chassis::DrivingMode_Name(mode);
  return false;
}

void HMIWorker::RunModeCommand(const std::string &command_name) {
  std::string current_mode;
  {
    RLock rlock(status_mutex_);
    current_mode = status_.current_mode();
  }
  const Mode &mode_conf = config_.modes().at(current_mode);
  if (command_name == "start" || command_name == "stop") {
    // Run the command on all live modules.
    for (const auto &module : mode_conf.live_modules()) {
      RunModuleCommand(module, command_name);
    }
  }
}

void HMIWorker::ChangeToMap(const std::string &map_name,
                            MapService *map_service) {
  const auto *map_dir = FindOrNull(config_.available_maps(), map_name);
  if (map_dir == nullptr) {
    AERROR << "Unknown map " << map_name;
    return;
  }

  {
    // Update current_map status.
    WLock wlock(status_mutex_);
    if (status_.current_map() == map_name) {
      return;
    }
    status_.set_current_map(map_name);
  }
  apollo::common::KVDB::Put("apollo:dreamview:map", map_name);

  SetGlobalFlag("map_dir", *map_dir, &FLAGS_map_dir);
  // Also reload simulation map.
  CHECK(map_service->ReloadMap(true)) << "Failed to load map from " << *map_dir;
  RunModeCommand("stop");
}

void HMIWorker::ChangeToVehicle(const std::string &vehicle_name) {
  const auto *vehicle = FindOrNull(config_.available_vehicles(), vehicle_name);
  if (vehicle == nullptr) {
    AERROR << "Unknown vehicle " << vehicle_name;
    return;
  }

  {
    // Update current_vehicle status.
    WLock wlock(status_mutex_);
    if (status_.current_vehicle() == vehicle_name) {
      return;
    }
    status_.set_current_vehicle(vehicle_name);
  }
  apollo::common::KVDB::Put("apollo:dreamview:vehicle", vehicle_name);

  CHECK(VehicleManager::instance()->UseVehicle(*vehicle));
  RunModeCommand("stop");
}

void HMIWorker::ChangeToMode(const std::string &mode_name) {
  if (!ContainsKey(config_.modes(), mode_name)) {
    AERROR << "Unknown mode " << mode_name;
    return;
  }

  std::string old_mode;
  {
    // Update current_mode status.
    WLock wlock(status_mutex_);
    old_mode = status_.current_mode();
    if (old_mode == mode_name) {
      return;
    }
    status_.set_current_mode(mode_name);
  }
  apollo::common::KVDB::Put("apollo:dreamview:mode", mode_name);

  const auto &old_modules = config_.modes().at(old_mode).live_modules();
  const bool use_navigation_mode = (mode_name == kNavigationModeName);
  SetGlobalFlag("use_navigation_mode", use_navigation_mode,
                &FLAGS_use_navigation_mode);
  // Now stop all old modules.
  for (const auto &module : old_modules) {
    RunModuleCommand(module, "stop");
  }
}

void HMIWorker::UpdateSystemStatus(const monitor::SystemStatus &system_status) {
  WLock wlock(status_mutex_);
  *status_.mutable_system_status() = system_status;
}

}  // namespace dreamview
}  // namespace apollo
<commit_msg>Data: Fix missing map if it's set by default.<commit_after>/******************************************************************************
 * Copyright 2018 The Apollo Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *****************************************************************************/

#include "modules/dreamview/backend/hmi/hmi_worker.h"

#include <vector>

#include "gflags/gflags.h"
#include "modules/common/adapters/adapter_manager.h"
#include "modules/common/kv_db/kv_db.h"
#include "modules/common/util/file.h"
#include "modules/common/util/map_util.h"
#include "modules/common/util/string_tokenizer.h"
#include "modules/common/util/string_util.h"
#include "modules/control/proto/pad_msg.pb.h"
#include "modules/data/util/info_collector.h"
#include "modules/dreamview/backend/common/dreamview_gflags.h"
#include "modules/dreamview/backend/hmi/vehicle_manager.h"

DEFINE_string(map_data_path, "/apollo/modules/map/data", "Path to map data.");

DEFINE_string(vehicle_data_path, "/apollo/modules/calibration/data",
              "Path to vehicle data.");

namespace apollo {
namespace dreamview {
namespace {

using apollo::canbus::Chassis;
using apollo::common::adapter::AdapterManager;
using apollo::common::util::ContainsKey;
using apollo::common::util::FindOrNull;
using apollo::common::util::StringTokenizer;
using apollo::control::DrivingAction;
using google::protobuf::Map;
using RLock = boost::shared_lock<boost::shared_mutex>;
using WLock = boost::unique_lock<boost::shared_mutex>;

constexpr char kNavigationModeName[] = "Navigation";

// Convert a string to be title-like. E.g.: "hello_world" -> "Hello World".
std::string TitleCase(const std::string &origin,
                      const std::string &delimiter = "_") {
  std::vector<std::string> parts = StringTokenizer::Split(origin, delimiter);
  for (auto &part : parts) {
    if (!part.empty()) {
      // Upper case the first char.
      part[0] = toupper(part[0]);
    }
  }

  return apollo::common::util::PrintIter(parts);
}

// List subdirs and return a dict of {subdir_title: subdir_path}.
Map<std::string, std::string> ListDirAsDict(const std::string &dir) {
  Map<std::string, std::string> result;
  const auto subdirs = apollo::common::util::ListSubDirectories(dir);
  for (const auto &subdir : subdirs) {
    const auto subdir_title = TitleCase(subdir);
    const auto subdir_path = apollo::common::util::StrCat(dir, "/", subdir);
    result.insert({subdir_title, subdir_path});
  }
  return result;
}

// Run a supported command of some component, return the ret code.
int RunComponentCommand(const Map<std::string, Component> &components,
                        const std::string &component_name,
                        const std::string &command_name) {
  const auto *component = FindOrNull(components, component_name);
  if (component == nullptr) {
    AERROR << "Cannot find component " << component_name;
    return -1;
  }
  const auto *cmd = FindOrNull(component->supported_commands(), command_name);
  if (cmd == nullptr) {
    AERROR << "Cannot find command " << component_name << "." << command_name;
    return -1;
  }
  AINFO << "Execute system command: " << *cmd;
  const int ret = std::system(cmd->c_str());

  AERROR_IF(ret != 0) << "Command returns " << ret << ": " << *cmd;
  return ret;
}

template <class FlagType, class ValueType>
void SetGlobalFlag(const std::string &flag_name, const ValueType &value,
                   FlagType *flag) {
  static constexpr char kGlobalFlagfile[] =
      "/apollo/modules/common/data/global_flagfile.txt";
  if (*flag != value) {
    *flag = value;
    // Overwrite global flagfile.
    std::ofstream fout(kGlobalFlagfile, std::ios_base::app);
    CHECK(fout) << "Fail to open global flagfile " << kGlobalFlagfile;
    fout << "\n--" << flag_name << "=" << value << std::endl;
  }
}

}  // namespace

HMIWorker::HMIWorker() {
  // Init HMIConfig.
  CHECK(common::util::GetProtoFromFile(FLAGS_hmi_config_filename, &config_))
      << "Unable to parse HMI config file " << FLAGS_hmi_config_filename;
  config_.set_docker_image(apollo::data::InfoCollector::GetDockerImage());

  // If the module path doesn't exist, remove it from list.
  auto *modules = config_.mutable_modules();
  for (auto iter = modules->begin(); iter != modules->end();) {
    const auto &conf = iter->second;
    if (conf.has_path() && !common::util::PathExists(conf.path())) {
      iter = modules->erase(iter);
    } else {
      ++iter;
    }
  }

  // Get available maps and vehicles by listing data directory.
  *config_.mutable_available_maps() = ListDirAsDict(FLAGS_map_data_path);
  *config_.mutable_available_vehicles() =
      ListDirAsDict(FLAGS_vehicle_data_path);
  config_.set_utm_zone_id(FLAGS_local_utm_zone_id);
  ADEBUG << "Loaded HMI config: " << config_.DebugString();

  // Init HMIStatus.
  const auto &modes = config_.modes();
  if (FLAGS_use_navigation_mode && ContainsKey(modes, kNavigationModeName)) {
    // If the FLAG_use_navigation_mode is set, set it in HMIStatus.
    status_.set_current_mode(kNavigationModeName);
  } else if (!ContainsKey(modes, status_.current_mode())) {
    CHECK(!modes.empty()) << "No available modes to run vehicle.";
    // If the default mode is unavailable, select the first one.
    status_.set_current_mode(modes.begin()->first);
  }

  // If the FLAGS_map_dir is set, set it in HMIStatus.
  if (!FLAGS_map_dir.empty()) {
    for (const auto &entry : config_.available_maps()) {
      // entry is (map_name, map_path)
      if (entry.second == FLAGS_map_dir) {
        status_.set_current_map(entry.first);
        apollo::common::KVDB::Put("apollo:dreamview:map", entry.first);
        break;
      }
    }
  }
}

bool HMIWorker::Trigger(const HMIAction action) {
  AINFO << "HMIAction " << HMIAction_Name(action) << " was triggered!";
  switch (action) {
    case HMIAction::SETUP:
      RunModeCommand("start");
      break;
    case HMIAction::AUTO_MODE:
      return ChangeToDrivingMode(Chassis::COMPLETE_AUTO_DRIVE);
    case HMIAction::DISENGAGE:
      return ChangeToDrivingMode(Chassis::COMPLETE_MANUAL);
    default:
      AERROR << "HMIAction not implemented, yet!";
      return false;
  }
  return true;
}

int HMIWorker::RunModuleCommand(const std::string &module,
                                const std::string &command) {
  return RunComponentCommand(config_.modules(), module, command);
}

int HMIWorker::RunHardwareCommand(const std::string &hardware,
                                  const std::string &command) {
  return RunComponentCommand(config_.hardware(), hardware, command);
}

int HMIWorker::RunToolCommand(const std::string &tool,
                              const std::string &command) {
  return RunComponentCommand(config_.tools(), tool, command);
}

void HMIWorker::SubmitDriveEvent(const uint64_t event_time_ms,
                                 const std::string &event_msg) {
  apollo::common::DriveEvent drive_event;
  AdapterManager::FillDriveEventHeader("HMI", &drive_event);
  drive_event.mutable_header()->set_timestamp_sec(event_time_ms / 1000.0);
  drive_event.set_event(event_msg);
  AdapterManager::PublishDriveEvent(drive_event);
}

bool HMIWorker::ChangeToDrivingMode(const Chassis::DrivingMode mode) {
  // Always reset to MANUAL mode before changing to other mode.
  if (mode != Chassis::COMPLETE_MANUAL) {
    if (!ChangeToDrivingMode(Chassis::COMPLETE_MANUAL)) {
      AERROR << "Failed to reset to MANUAL mode before changing to "
             << Chassis::DrivingMode_Name(mode);
      return false;
    }
  }

  control::PadMessage pad;
  switch (mode) {
    case Chassis::COMPLETE_MANUAL:
      pad.set_action(DrivingAction::RESET);
      break;
    case Chassis::COMPLETE_AUTO_DRIVE:
      pad.set_action(DrivingAction::START);
      break;
    default:
      AFATAL << "Unknown action to change driving mode to "
             << Chassis::DrivingMode_Name(mode);
  }

  constexpr int kMaxTries = 3;
  constexpr auto kTryInterval = std::chrono::milliseconds(500);
  auto *chassis = CHECK_NOTNULL(AdapterManager::GetChassis());
  for (int i = 0; i < kMaxTries; ++i) {
    // Send driving action periodically until entering target driving mode.
    AdapterManager::FillPadHeader("HMI", &pad);
    AdapterManager::PublishPad(pad);

    std::this_thread::sleep_for(kTryInterval);
    chassis->Observe();
    if (chassis->Empty()) {
      AERROR << "No Chassis message received!";
    } else if (chassis->GetLatestObserved().driving_mode() == mode) {
      return true;
    }
  }
  AERROR << "Failed to change driving mode to "
         << Chassis::DrivingMode_Name(mode);
  return false;
}

void HMIWorker::RunModeCommand(const std::string &command_name) {
  std::string current_mode;
  {
    RLock rlock(status_mutex_);
    current_mode = status_.current_mode();
  }
  const Mode &mode_conf = config_.modes().at(current_mode);
  if (command_name == "start" || command_name == "stop") {
    // Run the command on all live modules.
    for (const auto &module : mode_conf.live_modules()) {
      RunModuleCommand(module, command_name);
    }
  }
}

void HMIWorker::ChangeToMap(const std::string &map_name,
                            MapService *map_service) {
  const auto *map_dir = FindOrNull(config_.available_maps(), map_name);
  if (map_dir == nullptr) {
    AERROR << "Unknown map " << map_name;
    return;
  }

  {
    // Update current_map status.
    WLock wlock(status_mutex_);
    if (status_.current_map() == map_name) {
      return;
    }
    status_.set_current_map(map_name);
  }
  apollo::common::KVDB::Put("apollo:dreamview:map", map_name);

  SetGlobalFlag("map_dir", *map_dir, &FLAGS_map_dir);
  // Also reload simulation map.
  CHECK(map_service->ReloadMap(true)) << "Failed to load map from " << *map_dir;
  RunModeCommand("stop");
}

void HMIWorker::ChangeToVehicle(const std::string &vehicle_name) {
  const auto *vehicle = FindOrNull(config_.available_vehicles(), vehicle_name);
  if (vehicle == nullptr) {
    AERROR << "Unknown vehicle " << vehicle_name;
    return;
  }

  {
    // Update current_vehicle status.
    WLock wlock(status_mutex_);
    if (status_.current_vehicle() == vehicle_name) {
      return;
    }
    status_.set_current_vehicle(vehicle_name);
  }
  apollo::common::KVDB::Put("apollo:dreamview:vehicle", vehicle_name);

  CHECK(VehicleManager::instance()->UseVehicle(*vehicle));
  RunModeCommand("stop");
}

void HMIWorker::ChangeToMode(const std::string &mode_name) {
  if (!ContainsKey(config_.modes(), mode_name)) {
    AERROR << "Unknown mode " << mode_name;
    return;
  }

  std::string old_mode;
  {
    // Update current_mode status.
    WLock wlock(status_mutex_);
    old_mode = status_.current_mode();
    if (old_mode == mode_name) {
      return;
    }
    status_.set_current_mode(mode_name);
  }
  apollo::common::KVDB::Put("apollo:dreamview:mode", mode_name);

  const auto &old_modules = config_.modes().at(old_mode).live_modules();
  const bool use_navigation_mode = (mode_name == kNavigationModeName);
  SetGlobalFlag("use_navigation_mode", use_navigation_mode,
                &FLAGS_use_navigation_mode);
  // Now stop all old modules.
  for (const auto &module : old_modules) {
    RunModuleCommand(module, "stop");
  }
}

void HMIWorker::UpdateSystemStatus(const monitor::SystemStatus &system_status) {
  WLock wlock(status_mutex_);
  *status_.mutable_system_status() = system_status;
}

}  // namespace dreamview
}  // namespace apollo
<|endoftext|>


<commit_before>// vim: set sts=2 sw=2 et:
// encoding: utf-8
//
// Copyleft 2011 RIME Developers
// License: GPLv3
//
// 2011-7-12 Zou xu <zouivex@gmail.com>
//
#include <functional>
#include <queue>
#include <utility>
#include <vector>
#include <boost/foreach.hpp>
#include <rime/dict/prism.h>
#include <rime/algo/syllabifier.h>

namespace rime {

typedef std::pair<size_t, SpellingType> Vertex;
typedef std::priority_queue<Vertex, std::vector<Vertex>, std::greater<Vertex> > VertexQueue;

int Syllabifier::BuildSyllableGraph(const std::string &input, Prism &prism, SyllableGraph *graph) {
  if (input.empty())
    return 0;

  size_t farthest = 0;
  VertexQueue queue;
  queue.push(Vertex(0, kNormalSpelling));  // start

  while (!queue.empty()) {
    Vertex vertex(queue.top());
    queue.pop();
    size_t current_pos = vertex.first;

    // record a visit to the vertex
    VertexMap::iterator it = graph->vertices.find(current_pos);
    if (it == graph->vertices.end())
      graph->vertices.insert(vertex);  // preferred spelling type comes first
    else
      continue;  // discard worse spelling types

    // see where we can go by advancing a syllable
    std::vector<Prism::Match> matches;
    prism.CommonPrefixSearch(input.substr(current_pos), &matches);
    if (!matches.empty()) {
      EndVertexMap &end_vertices(graph->edges[current_pos]);
      BOOST_FOREACH(const Prism::Match &m, matches) {
        if (m.length == 0) continue;
        size_t end_pos = current_pos + m.length;
        // consume trailing delimiters
        while (end_pos < input.length() &&
               delimiters_.find(input[end_pos]) != std::string::npos)
          ++end_pos;
        if (end_pos > farthest)
          farthest = end_pos;
        SpellingMap &spellings(end_vertices[end_pos]);
        SpellingType end_vertex_type = kInvalidSpelling;
        // when spelling algebra is enabled, a spelling evaluates to a set of syllables;
        // otherwise, it resembles exactly the syllable itself.
        SpellingAccessor accessor(prism.QuerySpelling(m.value));
        while (!accessor.exhausted()) {
          SyllableId syllable_id(accessor.syllable_id());
          SpellingProperties props(accessor.properties());
          // add a syllable with properties to the edge's spelling-to-syllable map
          spellings.insert(SpellingMap::value_type(syllable_id, props));
          if (props.type < end_vertex_type) {
            end_vertex_type = props.type;
          }
          accessor.Next();
        }
        // render the vertex type
        if (end_vertex_type < vertex.second) {
          end_vertex_type = vertex.second;
        }
        queue.push(Vertex(end_pos, end_vertex_type));
      }
    }
  }

  // remove stale vertices and edges
  std::set<int> good;
  good.insert(farthest);
  SpellingType last_type = graph->vertices[farthest];
  for (int i = farthest - 1; i >= 0; --i) {
    if (graph->vertices.find(i) == graph->vertices.end())
      continue;
    // remove stale edges
    for (EndVertexMap::iterator j = graph->edges[i].begin();
         j != graph->edges[i].end(); ) {
      if (good.find(j->first) == good.end()) {
        // not connected
        graph->edges[i].erase(j++);
        continue;
      }
      for (SpellingMap::iterator k = j->second.begin();
           k != j->second.end(); ) {
        if (k->second.type > last_type)
          j->second.erase(k++);
        else
          ++k;
      }
      if (j->second.empty())
        graph->edges[i].erase(j++);
      else
        ++j;
    }
    if (graph->vertices[i] > last_type || graph->edges[i].empty()) {
      // remove stale vertex
      graph->vertices.erase(i);
      graph->edges.erase(i);
      continue;
    }
    // keep the valid vetex
    good.insert(i);
    if (graph->vertices[i] < last_type) {
      last_type = graph->vertices[i];
    }
  }

  if (enable_completion_ && farthest < input.length()) {
    const size_t kExpandSearchLimit = 512;
    std::vector<Prism::Match> keys;
    prism.ExpandSearch(input.substr(farthest), &keys, kExpandSearchLimit);
    if (!keys.empty()) {
      size_t current_pos = farthest;
      size_t end_pos = current_pos;
      size_t code_length = input.length() - farthest;
      EndVertexMap &end_vertices(graph->edges[current_pos]);
      BOOST_FOREACH(const Prism::Match &m, keys) {
        if (m.length < code_length) continue;
        end_pos = input.length();
        SpellingMap &spellings(end_vertices[end_pos]);
        // when spelling algebra is enabled, a spelling evaluates to a set of syllables;
        // otherwise, it resembles exactly the syllable itself.
        SpellingAccessor accessor(prism.QuerySpelling(m.value));
        while (!accessor.exhausted()) {
          SyllableId syllable_id(accessor.syllable_id());
          SpellingProperties props(accessor.properties());
          if (props.type > kNormalSpelling) continue;
          props.type = kCompletion;
          props.credibility *= 0.5;
          // add a syllable with properties to the edge's spelling-to-syllable map
          spellings.insert(SpellingMap::value_type(syllable_id, props));
          accessor.Next();
        }
        queue.push(Vertex(end_pos, kCompletion));
      }
      farthest = end_pos;
    }
  }

  graph->input_length = input.length();
  graph->interpreted_length = farthest;

  return farthest;
}

}  // namespace rime
<commit_msg>Syllablizer tricks; to be continued.<commit_after>// vim: set sts=2 sw=2 et:
// encoding: utf-8
//
// Copyleft 2011 RIME Developers
// License: GPLv3
//
// 2011-7-12 Zou xu <zouivex@gmail.com>
//
#include <functional>
#include <queue>
#include <utility>
#include <vector>
#include <boost/foreach.hpp>
#include <rime/dict/prism.h>
#include <rime/algo/syllabifier.h>

namespace rime {

typedef std::pair<size_t, SpellingType> Vertex;
typedef std::priority_queue<Vertex, std::vector<Vertex>, std::greater<Vertex> > VertexQueue;

int Syllabifier::BuildSyllableGraph(const std::string &input, Prism &prism, SyllableGraph *graph) {
  if (input.empty())
    return 0;

  size_t farthest = 0;
  VertexQueue queue;
  queue.push(Vertex(0, kNormalSpelling));  // start

  while (!queue.empty()) {
    Vertex vertex(queue.top());
    queue.pop();
    size_t current_pos = vertex.first;

    // record a visit to the vertex
    VertexMap::iterator it = graph->vertices.find(current_pos);
    if (it == graph->vertices.end())
      graph->vertices.insert(vertex);  // preferred spelling type comes first
    else
      continue;  // discard worse spelling types

    // see where we can go by advancing a syllable
    std::vector<Prism::Match> matches;
    prism.CommonPrefixSearch(input.substr(current_pos), &matches);
    if (!matches.empty()) {
      EndVertexMap &end_vertices(graph->edges[current_pos]);
      BOOST_FOREACH(const Prism::Match &m, matches) {
        if (m.length == 0) continue;
        size_t end_pos = current_pos + m.length;
        // consume trailing delimiters
        while (end_pos < input.length() &&
               delimiters_.find(input[end_pos]) != std::string::npos)
          ++end_pos;
        if (end_pos > farthest)
          farthest = end_pos;
        SpellingMap &spellings(end_vertices[end_pos]);
        SpellingType end_vertex_type = kInvalidSpelling;
        // for the same end vertex, favor longer spellings (possibly shorter path)
        double distance_factor = 1.0;
        if (graph->vertices.find(end_pos) != graph->vertices.end())
          distance_factor = 0.5;
        // when spelling algebra is enabled, a spelling evaluates to a set of syllables;
        // otherwise, it resembles exactly the syllable itself.
        SpellingAccessor accessor(prism.QuerySpelling(m.value));
        while (!accessor.exhausted()) {
          SyllableId syllable_id(accessor.syllable_id());
          SpellingProperties props(accessor.properties());
          props.credibility *= distance_factor;
          // add a syllable with properties to the edge's spelling-to-syllable map
          spellings.insert(SpellingMap::value_type(syllable_id, props));
          if (props.type < end_vertex_type) {
            end_vertex_type = props.type;
          }
          accessor.Next();
        }
        // render the vertex type
        if (end_vertex_type < vertex.second) {
          end_vertex_type = vertex.second;
        }
        queue.push(Vertex(end_pos, end_vertex_type));
      }
    }
  }

  // remove stale vertices and edges
  std::set<int> good;
  good.insert(farthest);
  SpellingType last_type = graph->vertices[farthest];
  for (int i = farthest - 1; i >= 0; --i) {
    if (graph->vertices.find(i) == graph->vertices.end())
      continue;
    // remove stale edges
    for (EndVertexMap::iterator j = graph->edges[i].begin();
         j != graph->edges[i].end(); ) {
      if (good.find(j->first) == good.end()) {
        // not connected
        graph->edges[i].erase(j++);
        continue;
      }
      for (SpellingMap::iterator k = j->second.begin();
           k != j->second.end(); ) {
        if (k->second.type > last_type)
          j->second.erase(k++);
        else
          ++k;
      }
      if (j->second.empty())
        graph->edges[i].erase(j++);
      else
        ++j;
    }
    if (graph->vertices[i] > last_type || graph->edges[i].empty()) {
      // remove stale vertex
      graph->vertices.erase(i);
      graph->edges.erase(i);
      continue;
    }
    // keep the valid vetex
    good.insert(i);
    if (graph->vertices[i] < last_type) {
      last_type = graph->vertices[i];
    }
  }

  if (enable_completion_ && farthest < input.length()) {
    const size_t kExpandSearchLimit = 512;
    std::vector<Prism::Match> keys;
    prism.ExpandSearch(input.substr(farthest), &keys, kExpandSearchLimit);
    if (!keys.empty()) {
      size_t current_pos = farthest;
      size_t end_pos = current_pos;
      size_t code_length = input.length() - farthest;
      EndVertexMap &end_vertices(graph->edges[current_pos]);
      BOOST_FOREACH(const Prism::Match &m, keys) {
        if (m.length < code_length) continue;
        end_pos = input.length();
        SpellingMap &spellings(end_vertices[end_pos]);
        // when spelling algebra is enabled, a spelling evaluates to a set of syllables;
        // otherwise, it resembles exactly the syllable itself.
        SpellingAccessor accessor(prism.QuerySpelling(m.value));
        while (!accessor.exhausted()) {
          SyllableId syllable_id(accessor.syllable_id());
          SpellingProperties props(accessor.properties());
          if (props.type > kNormalSpelling) continue;
          props.type = kCompletion;
          props.credibility *= 0.5;
          // add a syllable with properties to the edge's spelling-to-syllable map
          spellings.insert(SpellingMap::value_type(syllable_id, props));
          accessor.Next();
        }
        queue.push(Vertex(end_pos, kCompletion));
      }
      farthest = end_pos;
    }
  }

  graph->input_length = input.length();
  graph->interpreted_length = farthest;

  return farthest;
}

}  // namespace rime
<|endoftext|>


<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "stdafx.h"

#include <cstring>
#include <vector>

#include <zorba/error_list.h>
#include "diagnostics/xquery_exception.h"

#include "diagnostics/assert.h"
#include "diagnostics/dict.h"

#include "ascii_util.h"
#include "cxx_util.h"
#include "regex.h"
#include "stl_util.h"

#ifndef ZORBA_NO_UNICODE
# include <unicode/uversion.h>
#endif /* ZORBA_NO_UNICODE */

using namespace std;
U_NAMESPACE_USE

#define bs_c "\\p{L}\\d.:\\p{M}-"       /* \c equivalent contents */
#define bs_i "\\p{L}_:"                 /* \i equivalent contents */

#ifndef U_ICU_VERSION_MAJOR_NUM
# error "U_ICU_VERSION_MAJOR_NUM not defined"
#endif /* U_ICU_VERSION_MAJOR_NUM */

#if U_ICU_VERSION_MAJOR_NUM < 4
//
// UREGEX_LITERAL is only in ICU since 4.0.  For earlier versions, we define it
// ourselves.  Of course it won't have any effect since it's not implemented in
// ICU, but we implement it ourselves anyway since, even though the constant is
// defined in 4.0, it's not actually implemented as of 4.4.
//
# define UREGEX_LITERAL 16
#endif /* U_ICU_VERSION_MAJOR_NUM */

namespace zorba {

///////////////////////////////////////////////////////////////////////////////

typedef uint32_t icu_flags_t;

static icu_flags_t convert_xquery_flags( char const *xq_flags ) {
  icu_flags_t icu_flags = 0;
  if ( xq_flags ) {
    for ( char const *f = xq_flags; *f; ++f ) {
      switch ( *f ) {
        case 'i': icu_flags |= UREGEX_CASE_INSENSITIVE; break;
        case 'm': icu_flags |= UREGEX_MULTILINE       ; break;
        case 'q': icu_flags |= UREGEX_LITERAL         ; break;
        case 's': icu_flags |= UREGEX_DOTALL          ; break;
        case 'x': icu_flags |= UREGEX_COMMENTS        ; break;
        default:
          throw XQUERY_EXCEPTION( err::FORX0001, ERROR_PARAMS( *f ) );
      }
    }
    if ( icu_flags & UREGEX_LITERAL ) {
      //
      // XQuery 3.0 F&O: 5.6.1.1: If [the 'q' flag] is used together with the
      // m, s, or x flag, that flag has no effect.
      //
      icu_flags &= ~(
        UREGEX_MULTILINE  | // 'm'
        UREGEX_DOTALL     | // 's'
        UREGEX_COMMENTS     // 'x'
      );
    }
  }
  return icu_flags;
}

#define INVALID_RE_EXCEPTION(...) \
  XQUERY_EXCEPTION( err::FORX0002, ERROR_PARAMS( __VA_ARGS__ ) )

void convert_xquery_re( zstring const &xq_re, zstring *icu_re,
                        char const *xq_flags ) {
  icu_flags_t const icu_flags = convert_xquery_flags( xq_flags );
  bool const i_flag = (icu_flags & UREGEX_CASE_INSENSITIVE) != 0;
  bool const q_flag = (icu_flags & UREGEX_LITERAL) != 0;
  bool const x_flag = (icu_flags & UREGEX_COMMENTS) != 0;

  icu_re->clear();
  icu_re->reserve( xq_re.length() );    // approximate

  bool got_backslash = false;
  bool in_char_class = false;           // within [...]

  bool in_backref = false;              // '\'[1-9][0-9]*
  unsigned backref_no = 0;
  unsigned open_cap_subs = 0;

  // capture subgroup: true = open; false = closed
  vector<bool> cap_sub;

  FOR_EACH( zstring, xq_c, xq_re ) {
    if ( got_backslash ) {
      if ( x_flag && !in_char_class && ascii::is_space( *xq_c ) ) {
        //
        // XQuery 3.0 F&O 5.6.1.1: If [the 'x' flag is] present, whitespace
        // characters ... in the regular expression are removed prior to
        // matching with one exception: whitespace characters within character
        // class expressions ... are not removed.
        //
        continue;
      }
      got_backslash = false;
      switch ( *xq_c ) {
        case 'c': // NameChar
          *icu_re += "[" bs_c "]";
          continue;
        case 'C': // ^\c
          *icu_re += "[^" bs_c "]";
          continue;
        case 'i': // initial NameChar
          *icu_re += "[" bs_i "]";
          continue;
        case 'I': // ^\i
          *icu_re += "[^" bs_i "]";
          continue;
        default:
          if ( ascii::is_digit( *xq_c ) ) {
            backref_no = *xq_c - '0';
            if ( !backref_no )          // \0 is illegal
              throw INVALID_RE_EXCEPTION( xq_re, ZED( BackRef0Illegal ) );
            if ( in_char_class ) {
              //
              // XQuery 3.0 F&O 5.6.1: Within a character class expression,
              // \ followed by a digit is invalid.
              //
              throw INVALID_RE_EXCEPTION(
                xq_re, ZED( BackRefIllegalInCharClass )
              );
            }
            in_backref = true;
          }
          *icu_re += '\\';
          break;
      }
    } else {
      if ( in_backref ) {
        //
        // XQuery 3.0 F&O 5.6.1: The construct \N where N is a single digit is
        // always recognized as a back-reference; if this is followed by
        // further digits, these digits are taken to be part of the back-
        // reference if and only if the resulting number NN is such that the
        // back-reference is preceded by NN or more unescaped opening
        // parentheses.
        //
        if ( cap_sub.size() > 9 && ascii::is_digit( *xq_c ) )
          backref_no = backref_no * 10 + (*xq_c - '0');
        else
          in_backref = false;
        //
        // XQuery 3.0 F&O 5.6.1: The regular expression is invalid if a back-
        // reference refers to a subexpression that does not exist or whose
        // closing right parenthesis occurs after the back-reference.
        //
        if ( backref_no > cap_sub.size() )
          throw INVALID_RE_EXCEPTION(
            xq_re, ZED( NonexistentBackRef_3 ), backref_no
          );
        if ( cap_sub[ backref_no - 1 ] )
          throw INVALID_RE_EXCEPTION(
            xq_re, ZED( NonClosedBackRef_3 ), backref_no
          );
      }
      switch ( *xq_c ) {
        case '\\':
          got_backslash = true;
          continue;
        case '(':
          if ( q_flag )
            *icu_re += '\\';
          else {
            ++open_cap_subs;
            cap_sub.push_back( true );
          }
          break;
        case ')':
          if ( q_flag )
            *icu_re += '\\';
          else {
            if ( !open_cap_subs )
              throw INVALID_RE_EXCEPTION( xq_re, ZED( UnbalancedChar ), ')' );
            cap_sub[ --open_cap_subs ] = false;
          }
          break;
        case '[':
          if ( q_flag )
            *icu_re += '\\';
          else
            in_char_class = true;
          break;
        case ']':
          if ( q_flag )
            *icu_re += '\\';
          else
            in_char_class = false;
          break;
        default:
          if ( x_flag && ascii::is_space( *xq_c ) ) {
            if ( !in_char_class )
              continue;
            //
            // This is similar to the above case for removing whitespace except
            // ICU removes *all* whitespace (even within character classes)
            // unless the whitespace is escaped; hence we have to escape it.
            //
            *icu_re += '\\';
          }
      }
    }
    *icu_re += *xq_c;
  } // FOR_EACH

  if ( i_flag ) {
    //
    // XQuery 3.0 F&O 5.6.1.1: All other constructs are unaffected by the "i"
    // flag.  For example, "\p{Lu}" continues to match upper-case letters only.
    //
    // However, ICU lower-cases everything for the 'i' flag; hence we have to
    // turn off the 'i' flag for just the \p{Lu}.
    //
    // Note that the "6" and "12" below are correct since "\\" represents a
    // single '\'.
    //
    ascii::replace_all( *icu_re, "\\p{Lu}", 6, "(?-i:\\p{Lu})", 12 );
  }

  //
  // XML Schema Part 2 F.1.1: [Unicode Database] groups code points into a
  // number of blocks such as Basic Latin (i.e., ASCII), Latin-1 Supplement,
  // Hangul Jamo, CJK Compatibility, etc. The set containing all characters
  // that have block name X (with all white space stripped out), can be
  // identified with a block escape \p{IsX}.
  //
  // However, ICU uses \p{InX} rather than \p{IsX}.
  //
  // Note that the "5" below is correct since "\\" represents a single '\'.
  //
  ascii::replace_all( *icu_re, "\\p{Is", 5, "\\p{In", 5 );
}

///////////////////////////////////////////////////////////////////////////////

#ifndef ZORBA_NO_UNICODE

namespace unicode {

void regex::compile( string const &u_pattern, char const *flags,
                     char const *pattern ) {
  icu_flags_t const icu_flags = convert_xquery_flags( flags ) & ~UREGEX_LITERAL;
  delete matcher_;
  UErrorCode status = U_ZERO_ERROR;
  matcher_ = new RegexMatcher( u_pattern, icu_flags, status );
  if ( U_FAILURE( status ) ) {
    delete matcher_;
    matcher_ = nullptr;

    zstring icu_error_key;
    if ( status > U_REGEX_ERROR_START && status < U_REGEX_ERROR_LIMIT ) {
      icu_error_key = ZED_PREFIX;
      icu_error_key += u_errorName( status );
    }
    throw XQUERY_EXCEPTION(
      err::FORX0002, ERROR_PARAMS( pattern, icu_error_key )
    );
  }
}

bool regex::match_part( string const &s ) {
  ZORBA_ASSERT( matcher_ );
  matcher_->reset( s );
  return matcher_->find() != 0;
}

bool regex::match_whole( string const &s ) {
  ZORBA_ASSERT( matcher_ );
  matcher_->reset( s );
  UErrorCode status = U_ZERO_ERROR;
  return matcher_->matches( status ) && U_SUCCESS( status );
}

bool regex::next( re_type_t re_type, string const &s, size_type *pos,
                  string *substring, bool *matched ) {
  ZORBA_ASSERT( matcher_ );
  ZORBA_ASSERT( pos );
  unicode::size_type const s_len = s.length();
  if ( *pos < s_len ) {
    matcher_->reset( s );
    UErrorCode status = U_ZERO_ERROR;
    if ( matcher_->find( *pos, status ) ) {
      size_type const end = matcher_->end( status );
      if ( substring ) {
        size_type const start = matcher_->start( status );
        switch ( re_type ) {
          case re_is_match:
            substring->setTo( s, start, end - start );
            break;
          case re_is_separator:
            substring->setTo( s, *pos, start - *pos );
            break;
        }
      }
      *pos = end;
      if ( matched )
        *matched = true;
      return true;
    }
    if ( re_type == re_is_separator ) {
      //
      // Special case: the RE did not match starting at pos, but there *is* a
      // last token.  For example, given the RE of "," and:
      //
      //    s   = "a,b,c"
      //    pos =  01234
      //
      // then calling next_token() with a pos of 4 will not match another ","
      // but we should return the final token after the last "," (the "c").
      //
      if ( substring )
        substring->setTo( s, *pos, s_len - *pos );
      *pos = s_len;
      if ( matched )
        *matched = false;
      return true;
    }
  }
  if ( matched )
    *matched = false;
  return false;
}

bool regex::replace_all( string const &in, string const &replacement,
                         string *out ) {
  ZORBA_ASSERT( matcher_ );
  ZORBA_ASSERT( out );
  matcher_->reset( in );
  UErrorCode status = U_ZERO_ERROR;
  *out = matcher_->replaceAll( replacement, status );
  return U_SUCCESS( status ) == TRUE;
}

bool regex::replace_all( char const *in, char const *replacement,
                         string *out ) {
  string u_in, u_replacement;
  return  to_string( in, &u_in ) &&
          to_string( replacement, &u_replacement ) &&
          replace_all( u_in, u_replacement, out );
}

void regex::set_string(const char* in, size_type len, size_type *utf8len)
{
  ZORBA_ASSERT( matcher_ );
  to_string( in, len, &s_in );
  if(utf8len)
    *utf8len = s_in.length();
  matcher_->reset( s_in );
}

bool regex::find_next_match(bool *reachedEnd)
{
  ZORBA_ASSERT( matcher_ );
  UBool retfind = matcher_->find();
  if(reachedEnd)
  {
#if U_ICU_VERSION_MAJOR_NUM >= 4
  *reachedEnd = matcher_->hitEnd() != 0;
#else
  *reachedEnd = true;
#endif
  }
return retfind != 0;
}

int regex::get_pattern_group_count()
{
  ZORBA_ASSERT( matcher_ );
  return matcher_->groupCount();
}

int regex::get_match_start(int groupId)
{
  UErrorCode  errcode = U_ZERO_ERROR;
  ZORBA_ASSERT( matcher_ );
  return matcher_->start(groupId, errcode);
}

int regex::get_match_end(int groupId)
{
  UErrorCode  errcode = U_ZERO_ERROR;
  ZORBA_ASSERT( matcher_ );
  return matcher_->end(groupId, errcode);
}

} // namespace unicode

#endif /* ZORBA_NO_UNICODE */

///////////////////////////////////////////////////////////////////////////////

} // namespace zorba
/* vim:set et sw=2 ts=2: */
<commit_msg>fixed process module tests. BUG 3316641 and 3284888<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
//#include "stdafx.h"

#include "regex.h"

#include <cstring>
#include <vector>

#include <zorba/error_list.h>
#include "diagnostics/xquery_exception.h"

#include "diagnostics/assert.h"
#include "diagnostics/dict.h"

#include "ascii_util.h"
#include "cxx_util.h"
#include "stl_util.h"

#ifndef ZORBA_NO_UNICODE
# include <unicode/uversion.h>
#endif /* ZORBA_NO_UNICODE */

using namespace std;
U_NAMESPACE_USE

#define bs_c "\\p{L}\\d.:\\p{M}-"       /* \c equivalent contents */
#define bs_i "\\p{L}_:"                 /* \i equivalent contents */

#ifndef U_ICU_VERSION_MAJOR_NUM
# error "U_ICU_VERSION_MAJOR_NUM not defined"
#endif /* U_ICU_VERSION_MAJOR_NUM */

#if U_ICU_VERSION_MAJOR_NUM < 4
//
// UREGEX_LITERAL is only in ICU since 4.0.  For earlier versions, we define it
// ourselves.  Of course it won't have any effect since it's not implemented in
// ICU, but we implement it ourselves anyway since, even though the constant is
// defined in 4.0, it's not actually implemented as of 4.4.
//
# define UREGEX_LITERAL 16
#endif /* U_ICU_VERSION_MAJOR_NUM */

namespace zorba {

///////////////////////////////////////////////////////////////////////////////

typedef uint32_t icu_flags_t;

static icu_flags_t convert_xquery_flags( char const *xq_flags ) {
  icu_flags_t icu_flags = 0;
  if ( xq_flags ) {
    for ( char const *f = xq_flags; *f; ++f ) {
      switch ( *f ) {
        case 'i': icu_flags |= UREGEX_CASE_INSENSITIVE; break;
        case 'm': icu_flags |= UREGEX_MULTILINE       ; break;
        case 'q': icu_flags |= UREGEX_LITERAL         ; break;
        case 's': icu_flags |= UREGEX_DOTALL          ; break;
        case 'x': icu_flags |= UREGEX_COMMENTS        ; break;
        default:
          throw XQUERY_EXCEPTION( err::FORX0001, ERROR_PARAMS( *f ) );
      }
    }
    if ( icu_flags & UREGEX_LITERAL ) {
      //
      // XQuery 3.0 F&O: 5.6.1.1: If [the 'q' flag] is used together with the
      // m, s, or x flag, that flag has no effect.
      //
      icu_flags &= ~(
        UREGEX_MULTILINE  | // 'm'
        UREGEX_DOTALL     | // 's'
        UREGEX_COMMENTS     // 'x'
      );
    }
  }
  return icu_flags;
}

#define INVALID_RE_EXCEPTION(...) \
  XQUERY_EXCEPTION( err::FORX0002, ERROR_PARAMS( __VA_ARGS__ ) )

void convert_xquery_re( zstring const &xq_re, zstring *icu_re,
                        char const *xq_flags ) {
  icu_flags_t const icu_flags = convert_xquery_flags( xq_flags );
  bool const i_flag = (icu_flags & UREGEX_CASE_INSENSITIVE) != 0;
  bool const q_flag = (icu_flags & UREGEX_LITERAL) != 0;
  bool const x_flag = (icu_flags & UREGEX_COMMENTS) != 0;

  icu_re->clear();
  icu_re->reserve( xq_re.length() );    // approximate

  bool got_backslash = false;
  bool in_char_class = false;           // within [...]

  bool in_backref = false;              // '\'[1-9][0-9]*
  unsigned backref_no = 0;
  unsigned open_cap_subs = 0;

  // capture subgroup: true = open; false = closed
  vector<bool> cap_sub;

  FOR_EACH( zstring, xq_c, xq_re ) {
    if ( got_backslash ) {
      if ( x_flag && !in_char_class && ascii::is_space( *xq_c ) ) {
        //
        // XQuery 3.0 F&O 5.6.1.1: If [the 'x' flag is] present, whitespace
        // characters ... in the regular expression are removed prior to
        // matching with one exception: whitespace characters within character
        // class expressions ... are not removed.
        //
        continue;
      }
      got_backslash = false;
      switch ( *xq_c ) {
        case 'c': // NameChar
          *icu_re += "[" bs_c "]";
          continue;
        case 'C': // ^\c
          *icu_re += "[^" bs_c "]";
          continue;
        case 'i': // initial NameChar
          *icu_re += "[" bs_i "]";
          continue;
        case 'I': // ^\i
          *icu_re += "[^" bs_i "]";
          continue;
        default:
          if ( ascii::is_digit( *xq_c ) ) {
            backref_no = *xq_c - '0';
            if ( !backref_no )          // \0 is illegal
              throw INVALID_RE_EXCEPTION( xq_re, ZED( BackRef0Illegal ) );
            if ( in_char_class ) {
              //
              // XQuery 3.0 F&O 5.6.1: Within a character class expression,
              // \ followed by a digit is invalid.
              //
              throw INVALID_RE_EXCEPTION(
                xq_re, ZED( BackRefIllegalInCharClass )
              );
            }
            in_backref = true;
          }
          *icu_re += '\\';
          break;
      }
    } else {
      if ( in_backref ) {
        //
        // XQuery 3.0 F&O 5.6.1: The construct \N where N is a single digit is
        // always recognized as a back-reference; if this is followed by
        // further digits, these digits are taken to be part of the back-
        // reference if and only if the resulting number NN is such that the
        // back-reference is preceded by NN or more unescaped opening
        // parentheses.
        //
        if ( cap_sub.size() > 9 && ascii::is_digit( *xq_c ) )
          backref_no = backref_no * 10 + (*xq_c - '0');
        else
          in_backref = false;
        //
        // XQuery 3.0 F&O 5.6.1: The regular expression is invalid if a back-
        // reference refers to a subexpression that does not exist or whose
        // closing right parenthesis occurs after the back-reference.
        //
        if ( backref_no > cap_sub.size() )
          throw INVALID_RE_EXCEPTION(
            xq_re, ZED( NonexistentBackRef_3 ), backref_no
          );
        if ( cap_sub[ backref_no - 1 ] )
          throw INVALID_RE_EXCEPTION(
            xq_re, ZED( NonClosedBackRef_3 ), backref_no
          );
      }
      switch ( *xq_c ) {
        case '\\':
          got_backslash = true;
          continue;
        case '(':
          if ( q_flag )
            *icu_re += '\\';
          else {
            ++open_cap_subs;
            cap_sub.push_back( true );
          }
          break;
        case ')':
          if ( q_flag )
            *icu_re += '\\';
          else {
            if ( !open_cap_subs )
              throw INVALID_RE_EXCEPTION( xq_re, ZED( UnbalancedChar ), ')' );
            cap_sub[ --open_cap_subs ] = false;
          }
          break;
        case '[':
          if ( q_flag )
            *icu_re += '\\';
          else
            in_char_class = true;
          break;
        case ']':
          if ( q_flag )
            *icu_re += '\\';
          else
            in_char_class = false;
          break;
        default:
          if ( x_flag && ascii::is_space( *xq_c ) ) {
            if ( !in_char_class )
              continue;
            //
            // This is similar to the above case for removing whitespace except
            // ICU removes *all* whitespace (even within character classes)
            // unless the whitespace is escaped; hence we have to escape it.
            //
            *icu_re += '\\';
          }
      }
    }
    *icu_re += *xq_c;
  } // FOR_EACH

  if ( i_flag ) {
    //
    // XQuery 3.0 F&O 5.6.1.1: All other constructs are unaffected by the "i"
    // flag.  For example, "\p{Lu}" continues to match upper-case letters only.
    //
    // However, ICU lower-cases everything for the 'i' flag; hence we have to
    // turn off the 'i' flag for just the \p{Lu}.
    //
    // Note that the "6" and "12" below are correct since "\\" represents a
    // single '\'.
    //
    ascii::replace_all( *icu_re, "\\p{Lu}", 6, "(?-i:\\p{Lu})", 12 );
  }

  //
  // XML Schema Part 2 F.1.1: [Unicode Database] groups code points into a
  // number of blocks such as Basic Latin (i.e., ASCII), Latin-1 Supplement,
  // Hangul Jamo, CJK Compatibility, etc. The set containing all characters
  // that have block name X (with all white space stripped out), can be
  // identified with a block escape \p{IsX}.
  //
  // However, ICU uses \p{InX} rather than \p{IsX}.
  //
  // Note that the "5" below is correct since "\\" represents a single '\'.
  //
  ascii::replace_all( *icu_re, "\\p{Is", 5, "\\p{In", 5 );
}

///////////////////////////////////////////////////////////////////////////////

#ifndef ZORBA_NO_UNICODE

namespace unicode {

void regex::compile( string const &u_pattern, char const *flags,
                     char const *pattern ) {
  icu_flags_t const icu_flags = convert_xquery_flags( flags ) & ~UREGEX_LITERAL;
  delete matcher_;
  UErrorCode status = U_ZERO_ERROR;
  matcher_ = new RegexMatcher( u_pattern, icu_flags, status );
  if ( U_FAILURE( status ) ) {
    delete matcher_;
    matcher_ = nullptr;

    zstring icu_error_key;
    if ( status > U_REGEX_ERROR_START && status < U_REGEX_ERROR_LIMIT ) {
      icu_error_key = ZED_PREFIX;
      icu_error_key += u_errorName( status );
    }
    throw XQUERY_EXCEPTION(
      err::FORX0002, ERROR_PARAMS( pattern, icu_error_key )
    );
  }
}

bool regex::match_part( string const &s ) {
  ZORBA_ASSERT( matcher_ );
  matcher_->reset( s );
  return matcher_->find() != 0;
}

bool regex::match_whole( string const &s ) {
  ZORBA_ASSERT( matcher_ );
  matcher_->reset( s );
  UErrorCode status = U_ZERO_ERROR;
  return matcher_->matches( status ) && U_SUCCESS( status );
}

bool regex::next( re_type_t re_type, string const &s, size_type *pos,
                  string *substring, bool *matched ) {
  ZORBA_ASSERT( matcher_ );
  ZORBA_ASSERT( pos );
  unicode::size_type const s_len = s.length();
  if ( *pos < s_len ) {
    matcher_->reset( s );
    UErrorCode status = U_ZERO_ERROR;
    if ( matcher_->find( *pos, status ) ) {
      size_type const end = matcher_->end( status );
      if ( substring ) {
        size_type const start = matcher_->start( status );
        switch ( re_type ) {
          case re_is_match:
            substring->setTo( s, start, end - start );
            break;
          case re_is_separator:
            substring->setTo( s, *pos, start - *pos );
            break;
        }
      }
      *pos = end;
      if ( matched )
        *matched = true;
      return true;
    }
    if ( re_type == re_is_separator ) {
      //
      // Special case: the RE did not match starting at pos, but there *is* a
      // last token.  For example, given the RE of "," and:
      //
      //    s   = "a,b,c"
      //    pos =  01234
      //
      // then calling next_token() with a pos of 4 will not match another ","
      // but we should return the final token after the last "," (the "c").
      //
      if ( substring )
        substring->setTo( s, *pos, s_len - *pos );
      *pos = s_len;
      if ( matched )
        *matched = false;
      return true;
    }
  }
  if ( matched )
    *matched = false;
  return false;
}

bool regex::replace_all( string const &in, string const &replacement,
                         string *out ) {
  ZORBA_ASSERT( matcher_ );
  ZORBA_ASSERT( out );
  matcher_->reset( in );
  UErrorCode status = U_ZERO_ERROR;
  *out = matcher_->replaceAll( replacement, status );
  return U_SUCCESS( status ) == TRUE;
}

bool regex::replace_all( char const *in, char const *replacement,
                         string *out ) {
  string u_in, u_replacement;
  return  to_string( in, &u_in ) &&
          to_string( replacement, &u_replacement ) &&
          replace_all( u_in, u_replacement, out );
}

void regex::set_string(const char* in, size_type len, size_type *utf8len)
{
  ZORBA_ASSERT( matcher_ );
  to_string( in, len, &s_in );
  if(utf8len)
    *utf8len = s_in.length();
  matcher_->reset( s_in );
}

bool regex::find_next_match(bool *reachedEnd)
{
  ZORBA_ASSERT( matcher_ );
  UBool retfind = matcher_->find();
  if(reachedEnd)
  {
#if U_ICU_VERSION_MAJOR_NUM >= 4
  *reachedEnd = matcher_->hitEnd() != 0;
#else
  *reachedEnd = true;
#endif
  }
return retfind != 0;
}

int regex::get_pattern_group_count()
{
  ZORBA_ASSERT( matcher_ );
  return matcher_->groupCount();
}

int regex::get_match_start(int groupId)
{
  UErrorCode  errcode = U_ZERO_ERROR;
  ZORBA_ASSERT( matcher_ );
  return matcher_->start(groupId, errcode);
}

int regex::get_match_end(int groupId)
{
  UErrorCode  errcode = U_ZERO_ERROR;
  ZORBA_ASSERT( matcher_ );
  return matcher_->end(groupId, errcode);
}

} // namespace unicode

#endif /* ZORBA_NO_UNICODE */

///////////////////////////////////////////////////////////////////////////////

} // namespace zorba
/* vim:set et sw=2 ts=2: */
<|endoftext|>


<commit_before>/** \file    map_ddc_and_rvk_to_ixtheo_notations.cc
 *  \brief   Map certain DDC and RVK categories to ixTheo notations and add them to field 652a.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2015-2018, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <iterator>
#include <unordered_map>
#include <cctype>
#include <cstdlib>
#include "MARC.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " [--verbose] marc_input marc_output ddc_to_ixtheo_notations_map "
              << "rvk_to_ixtheo_notations_map\n";
    std::exit(EXIT_FAILURE);
}

/** \class IxTheoMapper
 *  \brief Maps from a DDC or RVK hierarchy entry to an IxTheo notation.
 */
class IxTheoMapper {
    std::string from_hierarchy_;
    std::string to_ix_theo_notation_;
    std::vector<std::string> exclusions_;
public:
    explicit IxTheoMapper(const std::vector<std::string> &map_file_line);

    /** \brief Returns an IxTheo notation if we can match "hierarchy_classification".  O/w we return the empty string. */
    std::string map(const std::string &hierarchy_classification) const;
};

IxTheoMapper::IxTheoMapper(const std::vector<std::string> &map_file_line) {
    if (map_file_line.size() < 2)
        throw std::runtime_error("in IxTheoMapper::IxTheoMapper: need at least 2 elements in \"map_file_line\"!");
    from_hierarchy_ = map_file_line[0];
    to_ix_theo_notation_ = map_file_line[1];
    std::copy(map_file_line.begin() + 2, map_file_line.end(), std::back_inserter(exclusions_));
}

std::string IxTheoMapper::map(const std::string &hierarchy_classification) const {
    if (not StringUtil::StartsWith(hierarchy_classification, from_hierarchy_))
        return "";

    for (const auto &exclusion : exclusions_) {
        if (StringUtil::StartsWith(hierarchy_classification, exclusion))
            return "";
    }

    return to_ix_theo_notation_;
}

void LoadCSVFile(const std::string &filename, std::vector<IxTheoMapper> * const mappers) {
    DSVReader csv_reader(filename);
    std::vector<std::string> csv_values;
    while (csv_reader.readLine(&csv_values))
        mappers->emplace_back(csv_values);

    std::cerr << "Read " << mappers->size() << " mappings from \"" << filename << "\".\n";
}

void UpdateIxTheoNotations(const std::vector<IxTheoMapper> &mappers, const std::vector<std::string> &orig_values,
                           std::string * const ixtheo_notations_list)
{
    std::vector<std::string> ixtheo_notations_vector;
    StringUtil::Split(*ixtheo_notations_list, ':', &ixtheo_notations_vector);
    std::set<std::string> previously_assigned_notations(std::make_move_iterator(ixtheo_notations_vector.begin()),
                                                        std::make_move_iterator(ixtheo_notations_vector.end()));

    for (const auto &mapper : mappers) {
        for (const auto &orig_value : orig_values) {
            const std::string mapped_value(mapper.map(orig_value));
            if (not mapped_value.empty()
                and previously_assigned_notations.find(mapped_value) == previously_assigned_notations.end())
            {
                if (not ixtheo_notations_list->empty())
                    *ixtheo_notations_list += ':';
                *ixtheo_notations_list += mapped_value;
                previously_assigned_notations.insert(mapped_value);
            }
        }
    }
}

void ProcessRecords(MARC::Reader * const marc_reader, MARC::Writer * const marc_writer,
                    const std::vector<IxTheoMapper> &ddc_to_ixtheo_notation_mappers,
                    const std::vector<IxTheoMapper> &/*rvk_to_ixtheo_notation_mappers*/)
{
    unsigned count(0), records_with_ixtheo_notations(0), records_with_new_notations(0), skipped_group_count(0);
    while (MARC::Record record = marc_reader->read()) {
        ++count;

        std::string ixtheo_notations_list(record.getFirstSubfieldValue("652", 'a'));
        if (not ixtheo_notations_list.empty()) {
            ++records_with_ixtheo_notations;
            marc_writer->write(record);
            continue;
        }

        std::vector<std::string> ddc_values(record.getSubfieldValues("082", 'a'));
        if (ddc_values.empty()) {
            marc_writer->write(record);
            continue;
        }

        // "K" stands for children's literature and "B" stands for fiction, both of which we don't want to
        // import into IxTheo;
        if (std::find(ddc_values.cbegin(), ddc_values.cend(), "K") != ddc_values.cend()
            or std::find(ddc_values.cbegin(), ddc_values.cend(), "B") != ddc_values.cend())
        {
            ++skipped_group_count;
            marc_writer->write(record);
            continue;
        }

        // Many DDC's have superfluous backslashes which are non-standard and need to be removed before further
        // processing can take place:
        for (auto &ddc_value : ddc_values)
            StringUtil::RemoveChars("/", &ddc_value);

        UpdateIxTheoNotations(ddc_to_ixtheo_notation_mappers, ddc_values, &ixtheo_notations_list);
        if (not ixtheo_notations_list.empty())
            LOG_DEBUG(record.getControlNumber() + ": " + StringUtil::Join(ddc_values, ',') + " -> " + ixtheo_notations_list);

/*
        std::vector<std::string> rvk_values;
        int _084_index(record.getFieldIndex("084"));
        if (_084_index != -1) {
            const std::vector<std::string> &fields(record.getFields());
            while (_084_index < static_cast<ssize_t>(dir_entries.size()) and dir_entries[_084_index].getTag() == "084") {
                const Subfields subfields(fields[_084_index]);
                if (subfields.hasSubfieldWithValue('2', "rvk"))
                    rvk_values.emplace_back(subfields.getFirstSubfieldValue('a'));
                ++_084_index;
            }
        }
        UpdateIxTheoNotations(rvk_to_ixtheo_notation_mappers, rvk_values, &ixtheo_notations_list);
*/

        if (not ixtheo_notations_list.empty()) {
            ++records_with_new_notations;
            record.insertField("652", "  ""\x1F""a" + ixtheo_notations_list + "\x1F""bDDCoderRVK");
        }

        marc_writer->write(record);
    }

    std::cout << ::progname << ": Read " << count << " records.\n";
    std::cout << ::progname << ": " << records_with_ixtheo_notations << " records had Ixtheo notations.\n";
    std::cout << ::progname << ": " << records_with_new_notations << " records received new Ixtheo notations.\n";
    std::cout << ::progname << ": " << skipped_group_count
              << " records where skipped because they were in a group that we are not interested in.\n";
}

} // unnamed namespace

int Main(int argc, char **argv) {
    if (argc < 5)
        Usage();

    auto marc_reader(MARC::Reader::Factory(argv[1]));
    auto marc_writer(MARC::Writer::Factory(argv[2]));

    std::vector<IxTheoMapper> ddc_to_ixtheo_notation_mappers;
    LoadCSVFile(argv[3], &ddc_to_ixtheo_notation_mappers);

    std::vector<IxTheoMapper> rvk_to_ixtheo_notation_mappers;
    // The RVK notations were never provided to us by the librarians
    // So, we don't  use it at the moment
    //LoadCSVFile(argv[4], &rvk_to_ixtheo_notation_mappers);

    ProcessRecords(marc_reader.get(), marc_writer.get(), ddc_to_ixtheo_notation_mappers,
                   rvk_to_ixtheo_notation_mappers);
    return EXIT_SUCCESS;
}
<commit_msg>Use LOG_INFO<commit_after>/** \file    map_ddc_and_rvk_to_ixtheo_notations.cc
 *  \brief   Map certain DDC and RVK categories to ixTheo notations and add them to field 652a.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2015-2018, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <iterator>
#include <unordered_map>
#include <cctype>
#include <cstdlib>
#include "MARC.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " [--verbose] marc_input marc_output ddc_to_ixtheo_notations_map "
              << "rvk_to_ixtheo_notations_map\n";
    std::exit(EXIT_FAILURE);
}

/** \class IxTheoMapper
 *  \brief Maps from a DDC or RVK hierarchy entry to an IxTheo notation.
 */
class IxTheoMapper {
    std::string from_hierarchy_;
    std::string to_ix_theo_notation_;
    std::vector<std::string> exclusions_;
public:
    explicit IxTheoMapper(const std::vector<std::string> &map_file_line);

    /** \brief Returns an IxTheo notation if we can match "hierarchy_classification".  O/w we return the empty string. */
    std::string map(const std::string &hierarchy_classification) const;
};

IxTheoMapper::IxTheoMapper(const std::vector<std::string> &map_file_line) {
    if (map_file_line.size() < 2)
        throw std::runtime_error("in IxTheoMapper::IxTheoMapper: need at least 2 elements in \"map_file_line\"!");
    from_hierarchy_ = map_file_line[0];
    to_ix_theo_notation_ = map_file_line[1];
    std::copy(map_file_line.begin() + 2, map_file_line.end(), std::back_inserter(exclusions_));
}

std::string IxTheoMapper::map(const std::string &hierarchy_classification) const {
    if (not StringUtil::StartsWith(hierarchy_classification, from_hierarchy_))
        return "";

    for (const auto &exclusion : exclusions_) {
        if (StringUtil::StartsWith(hierarchy_classification, exclusion))
            return "";
    }

    return to_ix_theo_notation_;
}

void LoadCSVFile(const std::string &filename, std::vector<IxTheoMapper> * const mappers) {
    DSVReader csv_reader(filename);
    std::vector<std::string> csv_values;
    while (csv_reader.readLine(&csv_values))
        mappers->emplace_back(csv_values);

    LOG_INFO("Read " + std::to_string(mappers->size()) + " mappings from '" + filename + "'.");
}

void UpdateIxTheoNotations(const std::vector<IxTheoMapper> &mappers, const std::vector<std::string> &orig_values,
                           std::string * const ixtheo_notations_list)
{
    std::vector<std::string> ixtheo_notations_vector;
    StringUtil::Split(*ixtheo_notations_list, ':', &ixtheo_notations_vector);
    std::set<std::string> previously_assigned_notations(std::make_move_iterator(ixtheo_notations_vector.begin()),
                                                        std::make_move_iterator(ixtheo_notations_vector.end()));

    for (const auto &mapper : mappers) {
        for (const auto &orig_value : orig_values) {
            const std::string mapped_value(mapper.map(orig_value));
            if (not mapped_value.empty()
                and previously_assigned_notations.find(mapped_value) == previously_assigned_notations.end())
            {
                if (not ixtheo_notations_list->empty())
                    *ixtheo_notations_list += ':';
                *ixtheo_notations_list += mapped_value;
                previously_assigned_notations.insert(mapped_value);
            }
        }
    }
}

void ProcessRecords(MARC::Reader * const marc_reader, MARC::Writer * const marc_writer,
                    const std::vector<IxTheoMapper> &ddc_to_ixtheo_notation_mappers,
                    const std::vector<IxTheoMapper> &/*rvk_to_ixtheo_notation_mappers*/)
{
    unsigned count(0), records_with_ixtheo_notations(0), records_with_new_notations(0), skipped_group_count(0);
    while (MARC::Record record = marc_reader->read()) {
        ++count;

        std::string ixtheo_notations_list(record.getFirstSubfieldValue("652", 'a'));
        if (not ixtheo_notations_list.empty()) {
            ++records_with_ixtheo_notations;
            marc_writer->write(record);
            continue;
        }

        std::vector<std::string> ddc_values(record.getSubfieldValues("082", 'a'));
        if (ddc_values.empty()) {
            marc_writer->write(record);
            continue;
        }

        // "K" stands for children's literature and "B" stands for fiction, both of which we don't want to
        // import into IxTheo;
        if (std::find(ddc_values.cbegin(), ddc_values.cend(), "K") != ddc_values.cend()
            or std::find(ddc_values.cbegin(), ddc_values.cend(), "B") != ddc_values.cend())
        {
            ++skipped_group_count;
            marc_writer->write(record);
            continue;
        }

        // Many DDC's have superfluous backslashes which are non-standard and need to be removed before further
        // processing can take place:
        for (auto &ddc_value : ddc_values)
            StringUtil::RemoveChars("/", &ddc_value);

        UpdateIxTheoNotations(ddc_to_ixtheo_notation_mappers, ddc_values, &ixtheo_notations_list);
        if (not ixtheo_notations_list.empty())
            LOG_DEBUG(record.getControlNumber() + ": " + StringUtil::Join(ddc_values, ',') + " -> " + ixtheo_notations_list);

/*
        std::vector<std::string> rvk_values;
        int _084_index(record.getFieldIndex("084"));
        if (_084_index != -1) {
            const std::vector<std::string> &fields(record.getFields());
            while (_084_index < static_cast<ssize_t>(dir_entries.size()) and dir_entries[_084_index].getTag() == "084") {
                const Subfields subfields(fields[_084_index]);
                if (subfields.hasSubfieldWithValue('2', "rvk"))
                    rvk_values.emplace_back(subfields.getFirstSubfieldValue('a'));
                ++_084_index;
            }
        }
        UpdateIxTheoNotations(rvk_to_ixtheo_notation_mappers, rvk_values, &ixtheo_notations_list);
*/

        if (not ixtheo_notations_list.empty()) {
            ++records_with_new_notations;
            record.insertField("652", "  ""\x1F""a" + ixtheo_notations_list + "\x1F""bDDCoderRVK");
        }

        marc_writer->write(record);
    }

    LOG_INFO("Read " + std::to_string(count) + " records.");
    LOG_INFO(std::to_string(records_with_ixtheo_notations) + " records had Ixtheo notations.");
    LOG_INFO(std::to_string(records_with_new_notations) + " records received new Ixtheo notations.");
    LOG_INFO(std::to_string(skipped_group_count) + " records where skipped because they were in a group that we are not interested in.");
}

} // unnamed namespace

int Main(int argc, char **argv) {
    if (argc < 5)
        Usage();

    auto marc_reader(MARC::Reader::Factory(argv[1]));
    auto marc_writer(MARC::Writer::Factory(argv[2]));

    std::vector<IxTheoMapper> ddc_to_ixtheo_notation_mappers;
    LoadCSVFile(argv[3], &ddc_to_ixtheo_notation_mappers);

    std::vector<IxTheoMapper> rvk_to_ixtheo_notation_mappers;
    // The RVK notations were never provided to us by the librarians
    // So, we don't  use it at the moment
    //LoadCSVFile(argv[4], &rvk_to_ixtheo_notation_mappers);

    ProcessRecords(marc_reader.get(), marc_writer.get(), ddc_to_ixtheo_notation_mappers,
                   rvk_to_ixtheo_notation_mappers);
    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>//  illarionserver - server for the game Illarion
//  Copyright 2011 Illarion e.V.
//
//  This file is part of illarionserver.
//
//  illarionserver is free software: you can redistribute it and/or modify
//  it under the terms of the GNU Affero General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
//
//  illarionserver is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU Affero General Public License for more details.
//
//  You should have received a copy of the GNU Affero General Public License
//  along with illarionserver.  If not, see <http://www.gnu.org/licenses/>.

#include "NewClientView.hpp"
#include "Map.hpp"
#include "WorldMap.hpp"
#include <iostream>

NewClientView::~NewClientView() {}

NewClientView::NewClientView() : viewPosition(position(0,0,0)), exists(false), stripedir(dir_right), maxtiles(0) {
    for (int i = 0; i < 100; ++i) {
        mapStripe[i] = nullptr;
    }
}

void NewClientView::fillStripe(position pos, stripedirection dir, int length, const WorldMap &maps) {
    clearStripe();
    viewPosition = pos;
    stripedir = dir;
    readFields(length, maps);
}

void NewClientView::clearStripe() {
    for (int i = 0; i < 100; ++i) {
        mapStripe[i] = nullptr;
    }

    exists = false;
    viewPosition.x = 0;
    viewPosition.y = 0;
    viewPosition.z = 0;
    maxtiles = 0;
}

void NewClientView::readFields(int length, const WorldMap &maps) {
    int x = viewPosition.x;
    int y = viewPosition.y;
    int x_inc = (stripedir == dir_right) ? 1 : -1;

    const auto good_maps = maps.findAllMapsInRangeOf(0, length-1, (stripedir == dir_right) ? length-1 : 0, (stripedir == dir_right) ? 0 : length-1, viewPosition);

    if (!good_maps.empty()) {
        WorldMap::map_t map;
        int tmp_maxtiles = 1;

        for (int i = 0; i < length; ++i) {
            Field *field = nullptr;

            if (!map || !map->GetPToCFieldAt(field,x,y)) {
                map.reset();

                for (auto it = good_maps.begin(); it != good_maps.end(); ++it) {
                    if ((*it)->GetPToCFieldAt(field,x,y)) {
                        map = *it;
                        break;
                    }
                }
            }

            if (field)
                if (!field->isTransparent() || !field->items.empty()) {
                    exists = true;
                    mapStripe[i] = field;
                    maxtiles = tmp_maxtiles;
                }

            ++tmp_maxtiles;
            //increase x due to perspective
            x += x_inc;
            //increase y due to perspective
            ++y;
        }
    }
}

<commit_msg>Modernize NewClientView<commit_after>//  illarionserver - server for the game Illarion
//  Copyright 2011 Illarion e.V.
//
//  This file is part of illarionserver.
//
//  illarionserver is free software: you can redistribute it and/or modify
//  it under the terms of the GNU Affero General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
//
//  illarionserver is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU Affero General Public License for more details.
//
//  You should have received a copy of the GNU Affero General Public License
//  along with illarionserver.  If not, see <http://www.gnu.org/licenses/>.

#include "NewClientView.hpp"
#include "Map.hpp"
#include "WorldMap.hpp"
#include <iostream>

NewClientView::~NewClientView() {}

NewClientView::NewClientView()
    : viewPosition(position(0, 0, 0)), exists(false), stripedir(dir_right),
      maxtiles(0) {
    for (auto &field : mapStripe) {
        field = nullptr;
    }
}

void NewClientView::fillStripe(position pos, stripedirection dir, int length,
                               const WorldMap &maps) {
    clearStripe();
    viewPosition = pos;
    stripedir = dir;
    readFields(length, maps);
}

void NewClientView::clearStripe() {
    for (auto &field : mapStripe) {
        field = nullptr;
    }

    exists = false;
    viewPosition.x = 0;
    viewPosition.y = 0;
    viewPosition.z = 0;
    maxtiles = 0;
}

void NewClientView::readFields(int length, const WorldMap &maps) {
    int x = viewPosition.x;
    int y = viewPosition.y;
    int x_inc = (stripedir == dir_right) ? 1 : -1;

    const auto good_maps = maps.findAllMapsInRangeOf(
        0, length - 1, (stripedir == dir_right) ? length - 1 : 0,
        (stripedir == dir_right) ? 0 : length - 1, viewPosition);

    if (!good_maps.empty()) {
        WorldMap::map_t map;
        int tmp_maxtiles = 1;

        for (int i = 0; i < length; ++i) {
            Field *field = nullptr;

            if (!map || !map->GetPToCFieldAt(field,x,y)) {
                map.reset();

                for (const auto &good_map : good_maps) {
                    if (good_map->GetPToCFieldAt(field,x,y)) {
                        map = good_map;
                        break;
                    }
                }
            }

            if (field)
                if (!field->isTransparent() || !field->items.empty()) {
                    exists = true;
                    mapStripe[i] = field;
                    maxtiles = tmp_maxtiles;
                }

            ++tmp_maxtiles;
            //increase x due to perspective
            x += x_inc;
            //increase y due to perspective
            ++y;
        }
    }
}

<|endoftext|>


<commit_before>/** Copyright (C) 2016 European Spallation Source ERIC */

#include <cstring>
#include <common/Detector.h>
#include <common/EFUArgs.h>
#include <common/Producer.h>
#include <common/RingBuffer.h>
#include <cspec/CSPECChanConv.h>
#include <cspec/CSPECData.h>
#include <cspec/CSPECEvent.h>
#include <iostream>
#include <libs/include/SPSCFifo.h>
#include <libs/include/Socket.h>
#include <libs/include/TSCTimer.h>
#include <libs/include/Timer.h>
#include <memory>
#include <mutex>
#include <queue>
#include <stdio.h>
#include <unistd.h>

/** @file
 *
 *  @brief CSPEC Detector implementation
 */

using namespace std;
using namespace memory_sequential_consistent; // Lock free fifo

const int TSC_MHZ = 2900; // Not accurate, do not rely solely on this

/** ----------------------------------------------------- */
const char *classname = "CSPEC Detector";

class CSPEC : public Detector {
public:
  void input_thread(void *args);
  void processing_thread(void *args);
  void output_thread(void *args);

  static const int eth_buffer_max_entries = 100000;
  static const int eth_buffer_size = 9000;
  static const int event_buffer_max_entries = 300 * eth_buffer_max_entries;
  static const int event_buffer_size = 32;
  static const int kafka_buffer_size = 1000000;

private:
  /** Shared between input_thread and processing_thread*/
  CircularFifo<struct RingBuffer<eth_buffer_size>::Data *, eth_buffer_max_entries> input2proc_fifo;

  /** Shared between processing_thread and output_thread */
  CircularFifo<struct RingBuffer<event_buffer_size>::Data *, event_buffer_max_entries> proc2output_fifo;

  std::mutex eventq_mutex, cout_mutex;

  char kafkabuffer[kafka_buffer_size];
};

void CSPEC::input_thread(void *args) {
  EFUArgs *opts = (EFUArgs *)args;
  RingBuffer<eth_buffer_size> ringbuf(eth_buffer_max_entries);

  /** Connection setup */
  Socket::Endpoint local(opts->ip_addr.c_str(), opts->port);
  UDPServer cspecdata(local);
  cspecdata.buflen(opts->buflen);
  cspecdata.setbuffers(0, opts->rcvbuf);
  cspecdata.printbuffers();
  cspecdata.settimeout(0, 100000); // One tenth of a second

  /** Buffer and stats setup */

  uint64_t rx = 0;
  uint64_t rx_total = 0;
  uint64_t rxp = 0;
  uint64_t ioverflow = 0;
  int rdsize;

  Timer us_clock, stop_timer;
  TSCTimer report_timer;
  for (;;) {

    /** this is the processing step */
    struct RingBuffer<eth_buffer_size>::Data *data = ringbuf.getdatastruct();

    if ((rdsize = cspecdata.receive(data->buffer, ringbuf.getsize())) > 0) {
      rxp++;
      rx += rdsize;
      ringbuf.setdatalength(rdsize);

      if (input2proc_fifo.push(data) == false) {
        ioverflow++;
      } else {
        ringbuf.nextbuffer();
      }
    }

    /** This is the periodic reporting*/
    if (unlikely(
            (report_timer.timetsc() >= opts->updint * 1000000 * TSC_MHZ))) {
      auto usecs = us_clock.timeus();
      rx_total += rx;

      cout_mutex.lock();
      printf("%" PRIu64 " input     : %8.2f Mb/s, q1: %3d, rxpkt: %12" PRIu64
             ", rxbytes: %12" PRIu64 ", push errors: %" PRIu64 "\n",
             report_timer.timetsc(), rx * 8.0 / usecs, 0, rxp, rx_total,
             ioverflow);
      fflush(stdout);
      cout_mutex.unlock();

      us_clock.now();
      report_timer.now();
      rx = 0;

      if (stop_timer.timeus() >= opts->stopafter * 1000000) {
        std::cout << "stopping input thread, timeus " << stop_timer.timeus()
                  << std::endl;
        return;
      }
    }
  }
}

void CSPEC::processing_thread(void *args) {
  EFUArgs *opts = (EFUArgs *)args;
  RingBuffer<event_buffer_size> ringbuf(event_buffer_max_entries);

  struct RingBuffer<eth_buffer_size>::Data *data;

  uint64_t ierror = 0;
  uint64_t oerror = 0;
  uint64_t idata = 0;
  uint64_t iidle = 0;
  uint64_t idisc = 0;

  CSPECChanConv conv;
  conv.makewirecal(0, CSPECChanConv::adcsize - 1,
                   128); // Linear dummy wire look-up table
  conv.makegridcal(0, CSPECChanConv::adcsize - 1,
                   96); // Linear dummy grid look-up table

  MultiGridGeometry CSPEC(80, 160, 4, 16);

  // CSPECData dat(0, 0, &conv, &CSPEC); // no signal threshold
  CSPECData dat(&conv, &CSPEC); // Default signal thresholds

  Timer us_clock, stopafter_clock;
  TSCTimer report_timer;

  uint64_t eventcount = 0;
  while (1) {

    if ((input2proc_fifo.pop(data)) == false) {
      iidle++;
      usleep(10);
    } else {
      dat.receive(data->buffer, data->length);
      ierror += dat.error;
      idata += dat.elems;
      idisc += dat.input_filter();

      for (auto d : dat.data) {
        if (d.valid) {
          struct RingBuffer<event_buffer_size>::Data *data = ringbuf.getdatastruct();
          dat.createevent(d, data->buffer);
          if (proc2output_fifo.push(data) == false) {
            oerror++;
          } else {
            ringbuf.nextbuffer();
          }
        }
      }
    }

    /** This is the periodic reporting*/
    if (unlikely(
            (report_timer.timetsc() >= opts->updint * 1000000 * TSC_MHZ))) {
      auto usecs = us_clock.timeus();
      auto rate = idata - eventcount;
      cout_mutex.lock();
      printf("%" PRIu64 " processing: idle: %" PRIu64 ", errors: %" PRIu64
             ", discard: %" PRIu64 ", events: %" PRIu64 ", kevts/s: %" PRIu64
             " , push errors: %" PRIu64 "\n",
             report_timer.timetsc(), iidle, ierror, idisc, idata, rate*1000/usecs,
             oerror);
      fflush(stdout);
      cout_mutex.unlock();

      eventcount = idata;
      us_clock.now();
      report_timer.now();

      if (stopafter_clock.timeus() >= opts->stopafter * 1000000) {
        std::cout << "stopping processing thread, timeus " << std::endl;
        return;
      }
    }
  }
}

void CSPEC::output_thread(void *args) {
  EFUArgs *opts = (EFUArgs *)args;

#ifndef NOKAFKA
  Producer producer(opts->broker, true, "C-SPEC_detector");
#endif

  struct RingBuffer<event_buffer_size>::Data *data;
  Timer stop;
  TSCTimer report_timer2;

  uint64_t rxevents = 0;
  uint64_t produce = 0;
  uint64_t idle = 0;
  while (1) {
    if (proc2output_fifo.pop(data) == false) {
      idle++;
      usleep(10);
    } else {
      std::memcpy(kafkabuffer + produce, data->buffer, 8 /**< @todo not hardcode */ );
      rxevents++;
      produce += 12; /**< @todo should match actual data size */
    }

    /** Produce when enough data has been accumulated */
    if (produce >= kafka_buffer_size - 1000) {
      assert(produce < kafka_buffer_size);
#ifndef NOKAFKA
      producer.produce(kafkabuffer, kafka_buffer_size);
#endif
      produce = 0;
    }

    /** This is the periodic reporting*/

    if (unlikely(
            (report_timer2.timetsc() >= opts->updint * 1000000 * TSC_MHZ))) {
      cout_mutex.lock();
      printf("%" PRIu64 " output    : events: %" PRIu64 ", idle: %" PRIu64
             " \n",
             report_timer2.timetsc(), rxevents, idle);
      fflush(stdout);
      cout_mutex.unlock();

      report_timer2.now();

      if (stop.timeus() >= opts->stopafter * 1000000) {
        std::cout << "stopping output thread, timeus " << stop.timeus()
                  << std::endl;
        return;
      }
    }
  }
}

/** ----------------------------------------------------- */

class CSPECFactory : public DetectorFactory {
public:
  std::shared_ptr<Detector> create() {
    return std::shared_ptr<Detector>(new CSPEC);
  }
};

CSPECFactory Factory;
<commit_msg>reduced event ringbuffer to the expected size (12 bytes) from 32<commit_after>/** Copyright (C) 2016 European Spallation Source ERIC */

#include <cstring>
#include <common/Detector.h>
#include <common/EFUArgs.h>
#include <common/Producer.h>
#include <common/RingBuffer.h>
#include <cspec/CSPECChanConv.h>
#include <cspec/CSPECData.h>
#include <cspec/CSPECEvent.h>
#include <iostream>
#include <libs/include/SPSCFifo.h>
#include <libs/include/Socket.h>
#include <libs/include/TSCTimer.h>
#include <libs/include/Timer.h>
#include <memory>
#include <mutex>
#include <queue>
#include <stdio.h>
#include <unistd.h>

/** @file
 *
 *  @brief CSPEC Detector implementation
 */

using namespace std;
using namespace memory_sequential_consistent; // Lock free fifo

const int TSC_MHZ = 2900; // Not accurate, do not rely solely on this

/** ----------------------------------------------------- */
const char *classname = "CSPEC Detector";

class CSPEC : public Detector {
public:
  void input_thread(void *args);
  void processing_thread(void *args);
  void output_thread(void *args);

  static const int eth_buffer_max_entries = 100000;
  static const int eth_buffer_size = 9000;
  static const int event_buffer_max_entries = 300 * eth_buffer_max_entries;
  static const int event_buffer_size = 12;
  static const int kafka_buffer_size = 1000000;

private:
  /** Shared between input_thread and processing_thread*/
  CircularFifo<struct RingBuffer<eth_buffer_size>::Data *, eth_buffer_max_entries> input2proc_fifo;

  /** Shared between processing_thread and output_thread */
  CircularFifo<struct RingBuffer<event_buffer_size>::Data *, event_buffer_max_entries> proc2output_fifo;

  std::mutex eventq_mutex, cout_mutex;

  char kafkabuffer[kafka_buffer_size];
};

void CSPEC::input_thread(void *args) {
  EFUArgs *opts = (EFUArgs *)args;
  RingBuffer<eth_buffer_size> ringbuf(eth_buffer_max_entries);

  /** Connection setup */
  Socket::Endpoint local(opts->ip_addr.c_str(), opts->port);
  UDPServer cspecdata(local);
  cspecdata.buflen(opts->buflen);
  cspecdata.setbuffers(0, opts->rcvbuf);
  cspecdata.printbuffers();
  cspecdata.settimeout(0, 100000); // One tenth of a second

  /** Buffer and stats setup */

  uint64_t rx = 0;
  uint64_t rx_total = 0;
  uint64_t rxp = 0;
  uint64_t ioverflow = 0;
  int rdsize;

  Timer us_clock, stop_timer;
  TSCTimer report_timer;
  for (;;) {

    /** this is the processing step */
    struct RingBuffer<eth_buffer_size>::Data *data = ringbuf.getdatastruct();

    if ((rdsize = cspecdata.receive(data->buffer, ringbuf.getsize())) > 0) {
      rxp++;
      rx += rdsize;
      ringbuf.setdatalength(rdsize);

      if (input2proc_fifo.push(data) == false) {
        ioverflow++;
      } else {
        ringbuf.nextbuffer();
      }
    }

    /** This is the periodic reporting*/
    if (unlikely(
            (report_timer.timetsc() >= opts->updint * 1000000 * TSC_MHZ))) {
      auto usecs = us_clock.timeus();
      rx_total += rx;

      cout_mutex.lock();
      printf("%" PRIu64 " input     : %8.2f Mb/s, q1: %3d, rxpkt: %12" PRIu64
             ", rxbytes: %12" PRIu64 ", push errors: %" PRIu64 "\n",
             report_timer.timetsc(), rx * 8.0 / usecs, 0, rxp, rx_total,
             ioverflow);
      fflush(stdout);
      cout_mutex.unlock();

      us_clock.now();
      report_timer.now();
      rx = 0;

      if (stop_timer.timeus() >= opts->stopafter * 1000000) {
        std::cout << "stopping input thread, timeus " << stop_timer.timeus()
                  << std::endl;
        return;
      }
    }
  }
}

void CSPEC::processing_thread(void *args) {
  EFUArgs *opts = (EFUArgs *)args;
  RingBuffer<event_buffer_size> ringbuf(event_buffer_max_entries);

  struct RingBuffer<eth_buffer_size>::Data *data;

  uint64_t ierror = 0;
  uint64_t oerror = 0;
  uint64_t idata = 0;
  uint64_t iidle = 0;
  uint64_t idisc = 0;

  CSPECChanConv conv;
  conv.makewirecal(0, CSPECChanConv::adcsize - 1,
                   128); // Linear dummy wire look-up table
  conv.makegridcal(0, CSPECChanConv::adcsize - 1,
                   96); // Linear dummy grid look-up table

  MultiGridGeometry CSPEC(80, 160, 4, 16);

  // CSPECData dat(0, 0, &conv, &CSPEC); // no signal threshold
  CSPECData dat(&conv, &CSPEC); // Default signal thresholds

  Timer us_clock, stopafter_clock;
  TSCTimer report_timer;

  uint64_t eventcount = 0;
  while (1) {

    if ((input2proc_fifo.pop(data)) == false) {
      iidle++;
      usleep(10);
    } else {
      dat.receive(data->buffer, data->length);
      ierror += dat.error;
      idata += dat.elems;
      idisc += dat.input_filter();

      for (auto d : dat.data) {
        if (d.valid) {
          struct RingBuffer<event_buffer_size>::Data *data = ringbuf.getdatastruct();
          dat.createevent(d, data->buffer);
          if (proc2output_fifo.push(data) == false) {
            oerror++;
          } else {
            ringbuf.nextbuffer();
          }
        }
      }
    }

    /** This is the periodic reporting*/
    if (unlikely(
            (report_timer.timetsc() >= opts->updint * 1000000 * TSC_MHZ))) {
      auto usecs = us_clock.timeus();
      auto rate = idata - eventcount;
      cout_mutex.lock();
      printf("%" PRIu64 " processing: idle: %" PRIu64 ", errors: %" PRIu64
             ", discard: %" PRIu64 ", events: %" PRIu64 ", kevts/s: %" PRIu64
             " , push errors: %" PRIu64 "\n",
             report_timer.timetsc(), iidle, ierror, idisc, idata, rate*1000/usecs,
             oerror);
      fflush(stdout);
      cout_mutex.unlock();

      eventcount = idata;
      us_clock.now();
      report_timer.now();

      if (stopafter_clock.timeus() >= opts->stopafter * 1000000) {
        std::cout << "stopping processing thread, timeus " << std::endl;
        return;
      }
    }
  }
}

void CSPEC::output_thread(void *args) {
  EFUArgs *opts = (EFUArgs *)args;

#ifndef NOKAFKA
  Producer producer(opts->broker, true, "C-SPEC_detector");
#endif

  struct RingBuffer<event_buffer_size>::Data *data;
  Timer stop;
  TSCTimer report_timer2;

  uint64_t rxevents = 0;
  uint64_t produce = 0;
  uint64_t idle = 0;
  while (1) {
    if (proc2output_fifo.pop(data) == false) {
      idle++;
      usleep(10);
    } else {
      std::memcpy(kafkabuffer + produce, data->buffer, 8 /**< @todo not hardcode */ );
      rxevents++;
      produce += 12; /**< @todo should match actual data size */
    }

    /** Produce when enough data has been accumulated */
    if (produce >= kafka_buffer_size - 1000) {
      assert(produce < kafka_buffer_size);
#ifndef NOKAFKA
      producer.produce(kafkabuffer, kafka_buffer_size);
#endif
      produce = 0;
    }

    /** This is the periodic reporting*/

    if (unlikely(
            (report_timer2.timetsc() >= opts->updint * 1000000 * TSC_MHZ))) {
      cout_mutex.lock();
      printf("%" PRIu64 " output    : events: %" PRIu64 ", idle: %" PRIu64
             " \n",
             report_timer2.timetsc(), rxevents, idle);
      fflush(stdout);
      cout_mutex.unlock();

      report_timer2.now();

      if (stop.timeus() >= opts->stopafter * 1000000) {
        std::cout << "stopping output thread, timeus " << stop.timeus()
                  << std::endl;
        return;
      }
    }
  }
}

/** ----------------------------------------------------- */

class CSPECFactory : public DetectorFactory {
public:
  std::shared_ptr<Detector> create() {
    return std::shared_ptr<Detector>(new CSPEC);
  }
};

CSPECFactory Factory;
<|endoftext|>