<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <glog/logging.h>

#include <iostream>
#include <string>
#include <vector>

#include <mesos/resources.hpp>
#include <mesos/scheduler.hpp>

#include <process/clock.hpp>
#include <process/dispatch.hpp>
#include <process/process.hpp>
#include <process/time.hpp>

#include <stout/bytes.hpp>
#include <stout/flags.hpp>
#include <stout/option.hpp>
#include <stout/os.hpp>
#include <stout/path.hpp>
#include <stout/stringify.hpp>

using namespace mesos;
using namespace mesos::internal;

using std::string;

using process::Clock;

const double CPUS_PER_TASK = 0.1;

const double CPUS_PER_EXECUTOR = 0.1;
const int32_t MEM_PER_EXECUTOR = 64;

class Flags : public flags::FlagsBase
{
public:
  Flags()
  {
    add(&master,
        "master",
        "Master to connect to.");

    add(&task_memory_usage_limit,
        "task_memory_usage_limit",
        None(),
        "Maximum size, in bytes, of the task's memory usage.\n"
        "The task will attempt to occupy memory up until this limit.",
        static_cast<const Bytes*>(nullptr),
        [](const Bytes& value) -> Option<Error> {
          if (value.megabytes() < MEM_PER_EXECUTOR) {
            return Error(
                "Please use a --task_memory_usage_limit greater than " +
                stringify(MEM_PER_EXECUTOR) + " MB");
          }

          return None();
        });

    add(&task_memory,
        "task_memory",
        "How much memory the framework will require per task.\n"
        "If not specified, the task(s) will use all available memory in\n"
        "applicable offers.");

    add(&build_dir,
        "build_dir",
        "The build directory of Mesos. If set, the framework will assume\n"
        "that the executor, framework, and agent(s) all live on the same\n"
        "machine.");

    add(&executor_uri,
        "executor_uri",
        "URI the fetcher should use to get the executor.");

    add(&executor_command,
        "executor_command",
        "The command that should be used to start the executor.\n"
        "This will override the value set by `--build_dir`.");

    add(&checkpoint,
        "checkpoint",
        "Whether this framework should be checkpointed.\n",
        false);

    add(&long_running,
        "long_running",
        "Whether this framework should launch tasks repeatedly\n"
        "or exit after finishing a single task.",
        false);
  }

  string master;
  Bytes task_memory_usage_limit;
  Bytes task_memory;

  // Flags for specifying the executor binary.
  Option<string> build_dir;
  Option<string> executor_uri;
  Option<string> executor_command;

  bool checkpoint;
  bool long_running;
};

// Actor holding the business logic and metrics for the `BalloonScheduler`.
// See `BalloonScheduler` below for the intended behavior.
class BalloonSchedulerProcess : public process::Process<BalloonSchedulerProcess>
{
public:
  BalloonSchedulerProcess(
      const ExecutorInfo& _executor,
      const Flags& _flags)
    : executor(_executor),
      flags(_flags),
      taskActive(false),
      tasksLaunched(0),
      isRegistered(false)
  {
    start_time = Clock::now();
  }

  void registered()
  {
    isRegistered = true;
  }

  void disconnected()
  {
    isRegistered = false;
  }

  void resourceOffers(
      SchedulerDriver* driver,
      const std::vector<Offer>& offers)
  {
    static const Resources TASK_RESOURCES = Resources::parse(
        "cpus:" + stringify(CPUS_PER_TASK) +
        ";mem:" + stringify(flags.task_memory.megabytes())).get();

    static const Resources EXECUTOR_RESOURCES = Resources(executor.resources());

    foreach (const Offer& offer, offers) {
      Resources resources(offer.resources());

      // If there is an active task, or if the offer is not
      // big enough, reject the offer.
      if (taskActive || !resources.flatten().contains(
            TASK_RESOURCES + EXECUTOR_RESOURCES)) {
        Filters filters;
        filters.set_refuse_seconds(600);

        driver->declineOffer(offer.id(), filters);
        continue;
      }

      int taskId = tasksLaunched++;

      LOG(INFO) << "Starting task " << taskId;

      TaskInfo task;
      task.set_name("Balloon Task");
      task.mutable_task_id()->set_value(stringify(taskId));
      task.mutable_slave_id()->MergeFrom(offer.slave_id());
      task.mutable_resources()->CopyFrom(TASK_RESOURCES);
      task.set_data(stringify(flags.task_memory_usage_limit));

      task.mutable_executor()->CopyFrom(executor);
      task.mutable_executor()->mutable_executor_id()
        ->set_value(stringify(taskId));

      driver->launchTasks(offer.id(), {task});

      taskActive = true;
    }
  }

  void statusUpdate(SchedulerDriver* driver, const TaskStatus& status)
  {
    if (!flags.long_running) {
      if (status.state() == TASK_FAILED &&
          status.reason() == TaskStatus::REASON_CONTAINER_LIMITATION_MEMORY) {
        // NOTE: We expect TASK_FAILED when this scheduler is launched by the
        // balloon_framework_test.sh shell script. The abort here ensures the
        // script considers the test result as "PASS".
        driver->abort();
      } else if (status.state() == TASK_FAILED ||
          status.state() == TASK_FINISHED ||
          status.state() == TASK_KILLED ||
          status.state() == TASK_LOST ||
          status.state() == TASK_ERROR) {
        driver->stop();
      }
    }

    // TODO(josephw): Add some metrics for some cases below.
    switch (status.state()) {
      case TASK_FINISHED:
      case TASK_FAILED:
      case TASK_KILLED:
      case TASK_LOST:
      case TASK_ERROR:
        taskActive = false;
        break;
      default:
        break;
    }
  }

private:
  const ExecutorInfo executor;
  const Flags flags;
  bool taskActive;
  int tasksLaunched;

  process::Time start_time;
  double _uptime_secs()
  {
    return (Clock::now() - start_time).secs();
  }

  bool isRegistered;
  double _registered()
  {
    return isRegistered ? 1 : 0;
  }

  // TODO(josephw): Add some metrics here.
};

// This scheduler starts a single executor and task which gradually
// increases its memory footprint up to a limit.  Depending on the
// resource limits set for the container, the framework expects the
// executor to either finish successfully or be OOM-killed.
class BalloonScheduler : public Scheduler
{
public:
  BalloonScheduler(
      const ExecutorInfo& _executor,
      const Flags& _flags)
    : process(_executor, _flags)
  {
    process::spawn(process);
  }

  virtual ~BalloonScheduler()
  {
    process::terminate(process);
    process::wait(process);
  }

  virtual void registered(
      SchedulerDriver*,
      const FrameworkID& frameworkId,
      const MasterInfo&)
  {
    LOG(INFO) << "Registered with framework ID: " << frameworkId;

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::registered);
  }

  virtual void reregistered(SchedulerDriver*, const MasterInfo& masterInfo)
  {
    LOG(INFO) << "Reregistered";

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::registered);
  }

  virtual void disconnected(SchedulerDriver* driver)
  {
    LOG(INFO) << "Disconnected";

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::disconnected);
  }

  virtual void resourceOffers(
      SchedulerDriver* driver,
      const std::vector<Offer>& offers)
  {
    LOG(INFO) << "Resource offers received";

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::resourceOffers,
        driver,
        offers);
  }

  virtual void offerRescinded(
      SchedulerDriver* driver,
      const OfferID& offerId)
  {
    LOG(INFO) << "Offer rescinded";
  }

  virtual void statusUpdate(SchedulerDriver* driver, const TaskStatus& status)
  {
    LOG(INFO)
      << "Task " << status.task_id() << " in state "
      << TaskState_Name(status.state())
      << ", Source: " << status.source()
      << ", Reason: " << status.reason()
      << (status.has_message() ? ", Message: " + status.message() : "");

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::statusUpdate,
        driver,
        status);
  }

  virtual void frameworkMessage(
      SchedulerDriver* driver,
      const ExecutorID& executorId,
      const SlaveID& slaveId,
      const string& data)
  {
    LOG(INFO) << "Framework message: " << data;
  }

  virtual void slaveLost(SchedulerDriver* driver, const SlaveID& slaveId)
  {
    LOG(INFO) << "Agent lost: " << slaveId;
  }

  virtual void executorLost(
      SchedulerDriver* driver,
      const ExecutorID& executorId,
      const SlaveID& slaveId,
      int status)
  {
    LOG(INFO) << "Executor '" << executorId << "' lost on agent: " << slaveId;
  }

  virtual void error(SchedulerDriver* driver, const string& message)
  {
    LOG(INFO) << "Error message: " << message;
  }

private:
  BalloonSchedulerProcess process;
};

int main(int argc, char** argv)
{
  Flags flags;
  Try<flags::Warnings> load = flags.load("MESOS_", argc, argv);

  if (load.isError()) {
    EXIT(EXIT_FAILURE) << flags.usage(load.error());
  }

  const Resources resources = Resources::parse(
      "cpus:" + stringify(CPUS_PER_EXECUTOR) +
      ";mem:" + stringify(MEM_PER_EXECUTOR)).get();

  ExecutorInfo executor;
  executor.mutable_resources()->CopyFrom(resources);
  executor.set_name("Balloon Executor");
  executor.set_source("balloon_test");

  // Determine the command to run the executor based on three possibilities:
  //   1) `--executor_command` was set, which overrides the below cases.
  //   2) We are in the Mesos build directory, so the targeted executable
  //      is actually a libtool wrapper script.
  //   3) We have not detected the Mesos build directory, so assume the
  //      executor is in the same directory as the framework.
  string command;

  // Find this executable's directory to locate executor.
  if (flags.executor_command.isSome()) {
    command = flags.executor_command.get();
  } else if (flags.build_dir.isSome()) {
    command = path::join(
        flags.build_dir.get(), "src", "balloon-executor");
  } else {
    command = path::join(
        os::realpath(Path(argv[0]).dirname()).get(),
        "balloon-executor");
  }

  executor.mutable_command()->set_value(command);

  // Copy `--executor_uri` into the command.
  if (flags.executor_uri.isSome()) {
    mesos::CommandInfo::URI* uri = executor.mutable_command()->add_uris();
    uri->set_value(flags.executor_uri.get());
    uri->set_executable(true);
  }

  BalloonScheduler scheduler(executor, flags);

  // Log any flag warnings (after logging is initialized by the scheduler).
  foreach (const flags::Warning& warning, load->warnings) {
    LOG(WARNING) << warning.message;
  }

  FrameworkInfo framework;
  framework.set_user(os::user().get());
  framework.set_name("Balloon Framework (C++)");
  framework.set_checkpoint(flags.checkpoint);

  MesosSchedulerDriver* driver;

  // TODO(josephw): Refactor these into a common set of flags.
  Option<string> value = os::getenv("MESOS_AUTHENTICATE_FRAMEWORKS");
  if (value.isSome()) {
    LOG(INFO) << "Enabling authentication for the framework";

    value = os::getenv("DEFAULT_PRINCIPAL");
    if (value.isNone()) {
      EXIT(EXIT_FAILURE)
        << "Expecting authentication principal in the environment";
    }

    Credential credential;
    credential.set_principal(value.get());

    framework.set_principal(value.get());

    value = os::getenv("DEFAULT_SECRET");
    if (value.isNone()) {
      EXIT(EXIT_FAILURE)
        << "Expecting authentication secret in the environment";
    }

    credential.set_secret(value.get());

    driver = new MesosSchedulerDriver(
        &scheduler, framework, flags.master, credential);
  } else {
    framework.set_principal("balloon-framework-cpp");

    driver = new MesosSchedulerDriver(
        &scheduler, framework, flags.master);
  }

  int status = driver->run() == DRIVER_STOPPED ? 0 : 1;

  // Ensure that the driver process terminates.
  driver->stop();

  delete driver;
  return status;
}
<commit_msg>Added metrics to the balloon framework.<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <glog/logging.h>

#include <iostream>
#include <string>
#include <vector>

#include <mesos/resources.hpp>
#include <mesos/scheduler.hpp>

#include <process/clock.hpp>
#include <process/defer.hpp>
#include <process/dispatch.hpp>
#include <process/http.hpp>
#include <process/process.hpp>
#include <process/protobuf.hpp>
#include <process/time.hpp>

#include <process/metrics/counter.hpp>
#include <process/metrics/gauge.hpp>
#include <process/metrics/metrics.hpp>

#include <stout/bytes.hpp>
#include <stout/flags.hpp>
#include <stout/option.hpp>
#include <stout/os.hpp>
#include <stout/path.hpp>
#include <stout/stringify.hpp>

using namespace mesos;
using namespace mesos::internal;

using std::string;

using process::Clock;
using process::defer;

using process::metrics::Gauge;
using process::metrics::Counter;

const double CPUS_PER_TASK = 0.1;

const double CPUS_PER_EXECUTOR = 0.1;
const int32_t MEM_PER_EXECUTOR = 64;

class Flags : public flags::FlagsBase
{
public:
  Flags()
  {
    add(&master,
        "master",
        "Master to connect to.");

    add(&task_memory_usage_limit,
        "task_memory_usage_limit",
        None(),
        "Maximum size, in bytes, of the task's memory usage.\n"
        "The task will attempt to occupy memory up until this limit.",
        static_cast<const Bytes*>(nullptr),
        [](const Bytes& value) -> Option<Error> {
          if (value.megabytes() < MEM_PER_EXECUTOR) {
            return Error(
                "Please use a --task_memory_usage_limit greater than " +
                stringify(MEM_PER_EXECUTOR) + " MB");
          }

          return None();
        });

    add(&task_memory,
        "task_memory",
        "How much memory the framework will require per task.\n"
        "If not specified, the task(s) will use all available memory in\n"
        "applicable offers.");

    add(&build_dir,
        "build_dir",
        "The build directory of Mesos. If set, the framework will assume\n"
        "that the executor, framework, and agent(s) all live on the same\n"
        "machine.");

    add(&executor_uri,
        "executor_uri",
        "URI the fetcher should use to get the executor.");

    add(&executor_command,
        "executor_command",
        "The command that should be used to start the executor.\n"
        "This will override the value set by `--build_dir`.");

    add(&checkpoint,
        "checkpoint",
        "Whether this framework should be checkpointed.\n",
        false);

    add(&long_running,
        "long_running",
        "Whether this framework should launch tasks repeatedly\n"
        "or exit after finishing a single task.",
        false);
  }

  string master;
  Bytes task_memory_usage_limit;
  Bytes task_memory;

  // Flags for specifying the executor binary.
  Option<string> build_dir;
  Option<string> executor_uri;
  Option<string> executor_command;

  bool checkpoint;
  bool long_running;
};

// Actor holding the business logic and metrics for the `BalloonScheduler`.
// See `BalloonScheduler` below for the intended behavior.
class BalloonSchedulerProcess : public process::Process<BalloonSchedulerProcess>
{
public:
  BalloonSchedulerProcess(
      const ExecutorInfo& _executor,
      const Flags& _flags)
    : executor(_executor),
      flags(_flags),
      taskActive(false),
      tasksLaunched(0),
      isRegistered(false),
      metrics(*this)
  {
    start_time = Clock::now();
  }

  void registered()
  {
    isRegistered = true;
  }

  void disconnected()
  {
    isRegistered = false;
  }

  void resourceOffers(
      SchedulerDriver* driver,
      const std::vector<Offer>& offers)
  {
    static const Resources TASK_RESOURCES = Resources::parse(
        "cpus:" + stringify(CPUS_PER_TASK) +
        ";mem:" + stringify(flags.task_memory.megabytes())).get();

    static const Resources EXECUTOR_RESOURCES = Resources(executor.resources());

    foreach (const Offer& offer, offers) {
      Resources resources(offer.resources());

      // If there is an active task, or if the offer is not
      // big enough, reject the offer.
      if (taskActive || !resources.flatten().contains(
            TASK_RESOURCES + EXECUTOR_RESOURCES)) {
        Filters filters;
        filters.set_refuse_seconds(600);

        driver->declineOffer(offer.id(), filters);
        continue;
      }

      int taskId = tasksLaunched++;

      LOG(INFO) << "Starting task " << taskId;

      TaskInfo task;
      task.set_name("Balloon Task");
      task.mutable_task_id()->set_value(stringify(taskId));
      task.mutable_slave_id()->MergeFrom(offer.slave_id());
      task.mutable_resources()->CopyFrom(TASK_RESOURCES);
      task.set_data(stringify(flags.task_memory_usage_limit));

      task.mutable_executor()->CopyFrom(executor);
      task.mutable_executor()->mutable_executor_id()
        ->set_value(stringify(taskId));

      driver->launchTasks(offer.id(), {task});

      taskActive = true;
    }
  }

  void statusUpdate(SchedulerDriver* driver, const TaskStatus& status)
  {
    if (!flags.long_running) {
      if (status.state() == TASK_FAILED &&
          status.reason() == TaskStatus::REASON_CONTAINER_LIMITATION_MEMORY) {
        // NOTE: We expect TASK_FAILED when this scheduler is launched by the
        // balloon_framework_test.sh shell script. The abort here ensures the
        // script considers the test result as "PASS".
        driver->abort();
      } else if (status.state() == TASK_FAILED ||
          status.state() == TASK_FINISHED ||
          status.state() == TASK_KILLED ||
          status.state() == TASK_LOST ||
          status.state() == TASK_ERROR) {
        driver->stop();
      }
    }

    if (stringify(tasksLaunched - 1) != status.task_id().value()) {
      // We might receive messages from older tasks. Ignore them.
      LOG(INFO) << "Ignoring status update from older task "
                << status.task_id();
      return;
    }

    switch (status.state()) {
      case TASK_FINISHED:
        taskActive = false;
        ++metrics.tasks_finished;
        break;
      case TASK_FAILED:
        taskActive = false;
        if (status.reason() == TaskStatus::REASON_CONTAINER_LIMITATION_MEMORY) {
          ++metrics.tasks_oomed;
          break;
        }

        // NOTE: Fetching the executor (e.g. `--executor_uri`) may fail
        // occasionally if the URI is rate limited. This case is common
        // enough that it makes sense to track this failure metric separately.
        if (status.reason() == TaskStatus::REASON_CONTAINER_LAUNCH_FAILED) {
          ++metrics.launch_failures;
          break;
        }
      case TASK_KILLED:
      case TASK_LOST:
      case TASK_ERROR:
        taskActive = false;

        ++metrics.abnormal_terminations;
        break;
      default:
        break;
    }
  }

private:
  const ExecutorInfo executor;
  const Flags flags;
  bool taskActive;
  int tasksLaunched;

  process::Time start_time;
  double _uptime_secs()
  {
    return (Clock::now() - start_time).secs();
  }

  bool isRegistered;
  double _registered()
  {
    return isRegistered ? 1 : 0;
  }

  struct Metrics
  {
    Metrics(const BalloonSchedulerProcess& _scheduler)
      : uptime_secs(
            "balloon_framework/uptime_secs",
            defer(_scheduler, &BalloonSchedulerProcess::_uptime_secs)),
        registered(
            "balloon_framework/registered",
            defer(_scheduler, &BalloonSchedulerProcess::_registered)),
        tasks_finished("balloon_framework/tasks_finished"),
        tasks_oomed("balloon_framework/tasks_oomed"),
        launch_failures("balloon_framework/launch_failures"),
        abnormal_terminations("balloon_framework/abnormal_terminations")
    {
      process::metrics::add(uptime_secs);
      process::metrics::add(registered);
      process::metrics::add(tasks_finished);
      process::metrics::add(tasks_oomed);
      process::metrics::add(launch_failures);
      process::metrics::add(abnormal_terminations);
    }

    ~Metrics()
    {
      process::metrics::remove(uptime_secs);
      process::metrics::remove(registered);
      process::metrics::remove(tasks_finished);
      process::metrics::remove(tasks_oomed);
      process::metrics::remove(launch_failures);
      process::metrics::remove(abnormal_terminations);
    }

    process::metrics::Gauge uptime_secs;
    process::metrics::Gauge registered;

    process::metrics::Counter tasks_finished;
    process::metrics::Counter tasks_oomed;
    process::metrics::Counter launch_failures;
    process::metrics::Counter abnormal_terminations;
  } metrics;
};

// This scheduler starts a single executor and task which gradually
// increases its memory footprint up to a limit.  Depending on the
// resource limits set for the container, the framework expects the
// executor to either finish successfully or be OOM-killed.
class BalloonScheduler : public Scheduler
{
public:
  BalloonScheduler(
      const ExecutorInfo& _executor,
      const Flags& _flags)
    : process(_executor, _flags)
  {
    process::spawn(process);
  }

  virtual ~BalloonScheduler()
  {
    process::terminate(process);
    process::wait(process);
  }

  virtual void registered(
      SchedulerDriver*,
      const FrameworkID& frameworkId,
      const MasterInfo&)
  {
    LOG(INFO) << "Registered with framework ID: " << frameworkId;

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::registered);
  }

  virtual void reregistered(SchedulerDriver*, const MasterInfo& masterInfo)
  {
    LOG(INFO) << "Reregistered";

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::registered);
  }

  virtual void disconnected(SchedulerDriver* driver)
  {
    LOG(INFO) << "Disconnected";

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::disconnected);
  }

  virtual void resourceOffers(
      SchedulerDriver* driver,
      const std::vector<Offer>& offers)
  {
    LOG(INFO) << "Resource offers received";

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::resourceOffers,
        driver,
        offers);
  }

  virtual void offerRescinded(
      SchedulerDriver* driver,
      const OfferID& offerId)
  {
    LOG(INFO) << "Offer rescinded";
  }

  virtual void statusUpdate(SchedulerDriver* driver, const TaskStatus& status)
  {
    LOG(INFO)
      << "Task " << status.task_id() << " in state "
      << TaskState_Name(status.state())
      << ", Source: " << status.source()
      << ", Reason: " << status.reason()
      << (status.has_message() ? ", Message: " + status.message() : "");

    process::dispatch(
        &process,
        &BalloonSchedulerProcess::statusUpdate,
        driver,
        status);
  }

  virtual void frameworkMessage(
      SchedulerDriver* driver,
      const ExecutorID& executorId,
      const SlaveID& slaveId,
      const string& data)
  {
    LOG(INFO) << "Framework message: " << data;
  }

  virtual void slaveLost(SchedulerDriver* driver, const SlaveID& slaveId)
  {
    LOG(INFO) << "Agent lost: " << slaveId;
  }

  virtual void executorLost(
      SchedulerDriver* driver,
      const ExecutorID& executorId,
      const SlaveID& slaveId,
      int status)
  {
    LOG(INFO) << "Executor '" << executorId << "' lost on agent: " << slaveId;
  }

  virtual void error(SchedulerDriver* driver, const string& message)
  {
    LOG(INFO) << "Error message: " << message;
  }

private:
  BalloonSchedulerProcess process;
};

int main(int argc, char** argv)
{
  Flags flags;
  Try<flags::Warnings> load = flags.load("MESOS_", argc, argv);

  if (load.isError()) {
    EXIT(EXIT_FAILURE) << flags.usage(load.error());
  }

  const Resources resources = Resources::parse(
      "cpus:" + stringify(CPUS_PER_EXECUTOR) +
      ";mem:" + stringify(MEM_PER_EXECUTOR)).get();

  ExecutorInfo executor;
  executor.mutable_resources()->CopyFrom(resources);
  executor.set_name("Balloon Executor");
  executor.set_source("balloon_test");

  // Determine the command to run the executor based on three possibilities:
  //   1) `--executor_command` was set, which overrides the below cases.
  //   2) We are in the Mesos build directory, so the targeted executable
  //      is actually a libtool wrapper script.
  //   3) We have not detected the Mesos build directory, so assume the
  //      executor is in the same directory as the framework.
  string command;

  // Find this executable's directory to locate executor.
  if (flags.executor_command.isSome()) {
    command = flags.executor_command.get();
  } else if (flags.build_dir.isSome()) {
    command = path::join(
        flags.build_dir.get(), "src", "balloon-executor");
  } else {
    command = path::join(
        os::realpath(Path(argv[0]).dirname()).get(),
        "balloon-executor");
  }

  executor.mutable_command()->set_value(command);

  // Copy `--executor_uri` into the command.
  if (flags.executor_uri.isSome()) {
    mesos::CommandInfo::URI* uri = executor.mutable_command()->add_uris();
    uri->set_value(flags.executor_uri.get());
    uri->set_executable(true);
  }

  BalloonScheduler scheduler(executor, flags);

  // Log any flag warnings (after logging is initialized by the scheduler).
  foreach (const flags::Warning& warning, load->warnings) {
    LOG(WARNING) << warning.message;
  }

  FrameworkInfo framework;
  framework.set_user(os::user().get());
  framework.set_name("Balloon Framework (C++)");
  framework.set_checkpoint(flags.checkpoint);

  MesosSchedulerDriver* driver;

  // TODO(josephw): Refactor these into a common set of flags.
  Option<string> value = os::getenv("MESOS_AUTHENTICATE_FRAMEWORKS");
  if (value.isSome()) {
    LOG(INFO) << "Enabling authentication for the framework";

    value = os::getenv("DEFAULT_PRINCIPAL");
    if (value.isNone()) {
      EXIT(EXIT_FAILURE)
        << "Expecting authentication principal in the environment";
    }

    Credential credential;
    credential.set_principal(value.get());

    framework.set_principal(value.get());

    value = os::getenv("DEFAULT_SECRET");
    if (value.isNone()) {
      EXIT(EXIT_FAILURE)
        << "Expecting authentication secret in the environment";
    }

    credential.set_secret(value.get());

    driver = new MesosSchedulerDriver(
        &scheduler, framework, flags.master, credential);
  } else {
    framework.set_principal("balloon-framework-cpp");

    driver = new MesosSchedulerDriver(
        &scheduler, framework, flags.master);
  }

  int status = driver->run() == DRIVER_STOPPED ? 0 : 1;

  // Ensure that the driver process terminates.
  driver->stop();

  delete driver;
  return status;
}
<|endoftext|>


<commit_before>/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "Directory.h"
#include "utils/Charsets.h"
#include "utils/Filename.h"
#include "utils/Url.h"
#include "factory/IFileSystem.h"
#include "File.h"
#include "logging/Logger.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <windows.h>
#include <winapifamily.h>

namespace medialibrary
{

namespace fs
{

Directory::Directory( const std::string& mrl , factory::IFileSystem& fsFactory )
    : CommonDirectory( fsFactory )
{
    m_path = utils::file::toFolderPath( toAbsolute( utils::file::toLocalPath( mrl ) ) );
    assert( *m_path.crbegin() == '/' );
    m_mrl = utils::file::toMrl( m_path );
}

const std::string& Directory::mrl() const
{
    return m_mrl;
}

void Directory::read() const
{
#if WINAPI_FAMILY_PARTITION (WINAPI_PARTITION_DESKTOP)
    WIN32_FIND_DATA f;
    auto pattern = m_path + '*';
    auto wpattern = charset::ToWide( pattern.c_str() );
    auto h = FindFirstFile( wpattern.get(), &f );
    if ( h == INVALID_HANDLE_VALUE )
    {
        LOG_ERROR( "Failed to browse ", m_path );
        throw std::system_error( GetLastError(), std::generic_category(), "Failed to browse through directory" );
    }
    do
    {
        auto file = charset::FromWide( f.cFileName );
        if ( file[0] == '.' && strcasecmp( file.get(), ".nomedia" ) )
            continue;
        if ( ( f.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) != 0 )
            m_dirs.emplace_back( m_fsFactory.createDirectory( m_path +
                                        utils::url::encode( fullpath ) ) );
        else
        {
            m_files.emplace_back( std::make_shared<File>( m_path + file.get() ) );
        }
    } while ( FindNextFile( h, &f ) != 0 );
    FindClose( h );
#else
    // We must remove the trailing /
    // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
    // «Do not use a trailing backslash (\), which indicates the root directory of a drive»
    auto tmpPath = path.substr( 0, m_path.length() - 1 );
    auto wpath = charset::ToWide( tmpPath.c_str() );

    CREATEFILE2_EXTENDED_PARAMETERS params{};
    params.dwFileFlags = FILE_FLAG_BACKUP_SEMANTICS;
    auto handle = CreateFile2( wpath.get(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, &params );
    if ( handle == INVALID_HANDLE_VALUE )
    {
        LOG_ERROR( "Failed to open directory ", m_path );
        throw std::system_error( GetLastError(), std::generic_category(), "Failed to open directory" );
    }

    std::unique_ptr<typename std::remove_pointer<HANDLE>::type,
            decltype(&CloseHandle)> handlePtr( handle, &CloseHandle );

    // Allocating a 32 bytes buffer to contain the file name. If more is required, we'll allocate
    size_t buffSize = sizeof( FILE_FULL_DIR_INFO ) + 32;
    std::unique_ptr<FILE_FULL_DIR_INFO, void(*)(FILE_FULL_DIR_INFO*)> dirInfo(
                reinterpret_cast<FILE_FULL_DIR_INFO*>( malloc( buffSize ) ),
                [](FILE_FULL_DIR_INFO* ptr) { free( ptr ); } );
    if ( dirInfo == nullptr )
        throw std::bad_alloc();

    while ( true )
    {
        auto h = GetFileInformationByHandleEx( handle, FileFullDirectoryInfo, dirInfo.get(), buffSize );
        if ( h == 0 )
        {
            auto error = GetLastError();
            if ( error == ERROR_FILE_NOT_FOUND )
                break;
            else if ( error == ERROR_MORE_DATA )
            {
                buffSize *= 2;
                dirInfo.reset( reinterpret_cast<FILE_FULL_DIR_INFO*>( malloc( buffSize ) ) );
                if ( dirInfo == nullptr )
                    throw std::bad_alloc();
                continue;
            }
            LOG_ERROR( "Failed to browse ", m_path, ". GetLastError(): ", GetLastError() );
            throw std::system_error( GetLastError(), std::generic_category(), "Failed to browse through directory" );
        }

        auto file = charset::FromWide( dirInfo->FileName );
        if ( file[0] == '.' && strcasecmp( file.get(), ".nomedia" ) )
            continue;
        if ( ( dirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY ) != 0 )
            m_dirs.emplace_back( m_fsFactory.createDirectory( m_mrl + utils::url::encode( file.get() ) ) );
        else
            m_files.emplace_back( std::make_shared<File>( m_path + file.get()) );
    }
#endif
}

std::string Directory::toAbsolute( const std::string& path )
{
    TCHAR buff[MAX_PATH];
    auto wpath = charset::ToWide( path.c_str() );
    if ( GetFullPathName( wpath.get(), MAX_PATH, buff, nullptr ) == 0 )
    {
        LOG_ERROR( "Failed to convert ", path, " to absolute path" );
        throw std::system_error( GetLastError(), std::generic_category(), "Failed to convert to absolute path" );
    }
    auto upath = charset::FromWide( buff );
    return std::string( upath.get() );
}

}

}
<commit_msg>win32: Directory: Fix build<commit_after>/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "Directory.h"
#include "utils/Charsets.h"
#include "utils/Filename.h"
#include "utils/Url.h"
#include "factory/IFileSystem.h"
#include "File.h"
#include "logging/Logger.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <windows.h>
#include <winapifamily.h>

namespace medialibrary
{

namespace fs
{

Directory::Directory( const std::string& mrl , factory::IFileSystem& fsFactory )
    : CommonDirectory( fsFactory )
{
    m_path = utils::file::toFolderPath( toAbsolute( utils::file::toLocalPath( mrl ) ) );
    assert( *m_path.crbegin() == '/' );
    m_mrl = utils::file::toMrl( m_path );
}

const std::string& Directory::mrl() const
{
    return m_mrl;
}

void Directory::read() const
{
#if WINAPI_FAMILY_PARTITION (WINAPI_PARTITION_DESKTOP)
    WIN32_FIND_DATA f;
    auto pattern = m_path + '*';
    auto wpattern = charset::ToWide( pattern.c_str() );
    auto h = FindFirstFile( wpattern.get(), &f );
    if ( h == INVALID_HANDLE_VALUE )
    {
        LOG_ERROR( "Failed to browse ", m_path );
        throw std::system_error( GetLastError(), std::generic_category(), "Failed to browse through directory" );
    }
    do
    {
        auto file = charset::FromWide( f.cFileName );
        if ( file[0] == '.' && strcasecmp( file.get(), ".nomedia" ) )
            continue;
        auto fullpath = m_path + file.get();
        if ( ( f.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) != 0 )
            m_dirs.emplace_back( m_fsFactory.createDirectory( m_path +
                                        utils::url::encode( fullpath ) ) );
        else
            m_files.emplace_back( std::make_shared<File>( fullpath ) );
    } while ( FindNextFile( h, &f ) != 0 );
    FindClose( h );
#else
    // We must remove the trailing /
    // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx
    // «Do not use a trailing backslash (\), which indicates the root directory of a drive»
    auto tmpPath = path.substr( 0, m_path.length() - 1 );
    auto wpath = charset::ToWide( tmpPath.c_str() );

    CREATEFILE2_EXTENDED_PARAMETERS params{};
    params.dwFileFlags = FILE_FLAG_BACKUP_SEMANTICS;
    auto handle = CreateFile2( wpath.get(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, &params );
    if ( handle == INVALID_HANDLE_VALUE )
    {
        LOG_ERROR( "Failed to open directory ", m_path );
        throw std::system_error( GetLastError(), std::generic_category(), "Failed to open directory" );
    }

    std::unique_ptr<typename std::remove_pointer<HANDLE>::type,
            decltype(&CloseHandle)> handlePtr( handle, &CloseHandle );

    // Allocating a 32 bytes buffer to contain the file name. If more is required, we'll allocate
    size_t buffSize = sizeof( FILE_FULL_DIR_INFO ) + 32;
    std::unique_ptr<FILE_FULL_DIR_INFO, void(*)(FILE_FULL_DIR_INFO*)> dirInfo(
                reinterpret_cast<FILE_FULL_DIR_INFO*>( malloc( buffSize ) ),
                [](FILE_FULL_DIR_INFO* ptr) { free( ptr ); } );
    if ( dirInfo == nullptr )
        throw std::bad_alloc();

    while ( true )
    {
        auto h = GetFileInformationByHandleEx( handle, FileFullDirectoryInfo, dirInfo.get(), buffSize );
        if ( h == 0 )
        {
            auto error = GetLastError();
            if ( error == ERROR_FILE_NOT_FOUND )
                break;
            else if ( error == ERROR_MORE_DATA )
            {
                buffSize *= 2;
                dirInfo.reset( reinterpret_cast<FILE_FULL_DIR_INFO*>( malloc( buffSize ) ) );
                if ( dirInfo == nullptr )
                    throw std::bad_alloc();
                continue;
            }
            LOG_ERROR( "Failed to browse ", m_path, ". GetLastError(): ", GetLastError() );
            throw std::system_error( GetLastError(), std::generic_category(), "Failed to browse through directory" );
        }

        auto file = charset::FromWide( dirInfo->FileName );
        if ( file[0] == '.' && strcasecmp( file.get(), ".nomedia" ) )
            continue;
        if ( ( dirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY ) != 0 )
            m_dirs.emplace_back( m_fsFactory.createDirectory( m_mrl + utils::url::encode( file.get() ) ) );
        else
            m_files.emplace_back( std::make_shared<File>( m_path + file.get()) );
    }
#endif
}

std::string Directory::toAbsolute( const std::string& path )
{
    TCHAR buff[MAX_PATH];
    auto wpath = charset::ToWide( path.c_str() );
    if ( GetFullPathName( wpath.get(), MAX_PATH, buff, nullptr ) == 0 )
    {
        LOG_ERROR( "Failed to convert ", path, " to absolute path" );
        throw std::system_error( GetLastError(), std::generic_category(), "Failed to convert to absolute path" );
    }
    auto upath = charset::FromWide( buff );
    return std::string( upath.get() );
}

}

}
<|endoftext|>


<commit_before>#include "graph/expression_operators.h"
#include "layers/constructors.h"

#include "graph/node_operators.h"
#include "graph/node_operators_binary.h"
#include "graph/node_operators_unary.h"

namespace marian {

Expr debug(Expr a, const std::string& message) {
  a->debug(message);
  return a;
}

Expr logit(Expr a) {
  return Expression<LogitNodeOp>(a);
}

Expr relu(Expr a) {
  return Expression<ReLUNodeOp>(a);
}

Expr leakyrelu(Expr a) {
  return Expression<PReLUNodeOp>(0.01f, a);
}

Expr prelu(Expr a, float alpha) {
  return Expression<PReLUNodeOp>(alpha, a);
}

Expr log(Expr a) {
  return Expression<LogNodeOp>(a);
};

Expr exp(Expr a) {
  return Expression<ExpNodeOp>(a);
};

Expr swish(Expr a) {
  return Expression<SwishNodeOp>(a);
}

Expr operator-(Expr a) {
  return Expression<NegNodeOp>(a);
};

Expr softmax(Expr a, Expr mask) {
  return Expression<SoftmaxNodeOp>(a, mask);
}

Expr logsoftmax(Expr a) {
  return Expression<LogSoftmaxNodeOp>(a);
}

/*********************************************************/

Expr operator+(Expr a, Expr b) {
  return Expression<PlusNodeOp>(a, b);
}

Expr operator-(Expr a, Expr b) {
  return Expression<MinusNodeOp>(a, b);
}

Expr operator*(Expr a, Expr b) {
  return Expression<MultNodeOp>(a, b);
}

Expr operator/(Expr a, Expr b) {
  return Expression<DivNodeOp>(a, b);
}

/*********************************************************/

Expr operator+(Expr a, float b) {
  return Expression<ScalarAddNodeOp>(a, b);
}

Expr operator+(float a, Expr b) {
  return Expression<ScalarAddNodeOp>(b, a);
}

Expr operator-(Expr a, float b) {
  return Expression<ScalarAddNodeOp>(a, -b);
}

Expr operator-(float a, Expr b) {
  return Expression<ScalarAddNodeOp>(-b, a);
}

Expr operator*(float a, Expr b) {
  return Expression<ScalarMultNodeOp>(b, a);
}

Expr operator*(Expr a, float b) {
  return Expression<ScalarMultNodeOp>(a, b);
}

Expr operator/(Expr a, float b) {
  return Expression<ScalarMultNodeOp>(a, 1.f / b);
}

// Expr pow(float a, Expr b) {
//  return Expression<Scalar1PowNodeOp>(a, b);
//
//}
//
// Expr pow(Expr a, float b) {
//  return Expression<Scalar2PowNodeOp>(a, b);
//
//}
//
// Expr pow(Expr a, Expr b) {
//  return Expression<PowNodeOp>(a, b);
//}

/*********************************************************/

Expr concatenate(const std::vector<Expr>& concats, keywords::axis_k ax) {
  return Expression<ConcatenateNodeOp>(concats, ax);
}

Expr repeat(Expr a, size_t repeats, keywords::axis_k ax) {
  if(repeats == 1)
    return a;
  return concatenate(std::vector<Expr>(repeats, a), ax);
}

Expr reshape(Expr a, Shape shape) {
  return Expression<ReshapeNodeOp>(a, shape);
}

Expr atleast_1d(Expr a) {
  return atleast_nd(a, 1);
}

Expr atleast_2d(Expr a) {
  return atleast_nd(a, 2);
}

Expr atleast_3d(Expr a) {
  return atleast_nd(a, 3);
}

Expr atleast_4d(Expr a) {
  return atleast_nd(a, 4);
}

Expr atleast_nd(Expr a, size_t dims) {
  if(a->shape().size() >= dims)
    return a;

  Shape nShape;
  nShape.resize(dims);
  for(int i = 1; i <= a->shape().size(); ++i)
    nShape.set(-i, a->shape()[-i]);

  return reshape(a, nShape);
}

Expr flatten(Expr a) {
  Shape shape = {a->shape().elements()};
  return Expression<ReshapeNodeOp>(a, shape);
}

Expr flatten_2d(Expr a) {
  Shape shape = {a->shape().elements() / a->shape()[-1], a->shape()[-1]};

  return Expression<ReshapeNodeOp>(a, shape);
}

Expr rows(Expr a, const std::vector<size_t>& indices) {
  return Expression<RowsNodeOp>(a, indices);
}

Expr cols(Expr a, const std::vector<size_t>& indices) {
  return Expression<ColsNodeOp>(a, indices);
}

Expr select(Expr a, int axis, const std::vector<size_t>& indices) {
  return Expression<SelectNodeOp>(a, axis, indices);
}

Expr sum(Expr a, keywords::axis_k ax) {
  return Expression<SumNodeOp>(a, ax);
}

Expr mean(Expr a, keywords::axis_k ax) {
  return Expression<MeanNodeOp>(a, ax);
}

Expr scalar_product(Expr a, Expr b, keywords::axis_k ax) {
  return Expression<ScalarProductNodeOp>(a, b, ax);
}

Expr weighted_average(Expr in, Expr weights, keywords::axis_k ax) {
  auto p = scalar_product(in, weights, ax);
  auto s = sum(weights, ax);
  return p / s;
}

Expr dot(Expr a, Expr b, bool transA, bool transB, float scalar) {
  return Expression<DotNodeOp>(a, b, transA, transB, scalar);
}

Expr bdot(Expr a, Expr b, bool transA, bool transB, float scalar) {
  return Expression<DotBatchedNodeOp>(a, b, transA, transB, scalar);
}

Expr transpose(Expr a) {
  std::vector<int> axes(a->shape().size());
  for(int i = 0; i < axes.size(); ++i) {
    axes[i] = i;
  }
  if(axes.size() > 1) {
    axes[axes.size() - 1] = axes.size() - 2;
    axes[axes.size() - 2] = axes.size() - 1;
  }
  return Expression<TransposeNodeOp>(a, axes);
}

Expr transpose(Expr a, const std::vector<int>& axes) {
  return Expression<TransposeNodeOp>(a, axes);
}

Expr step(Expr a, int step, int axis) {
  return Expression<StepNodeOp>(a, step, axis);
}

Expr cross_entropy(Expr a, Expr b) {
  // auto sOrig = a->shape();
  // auto sOut = a->shape();
  // Shape sTemp({sOrig[0] * sOrig[2] * sOrig[3], sOrig[1], 1, 1});
  // sOut.set(1, 1);
  // return reshape(Expression<CrossEntropyNodeOp>(reshape(a, sTemp), b), sOut);

  return Expression<CrossEntropyNodeOp>(a, b);
}

Expr affine(Expr a, Expr b, Expr c, bool transA, bool transB, float scalar) {
  std::vector<Expr> nodes = {a, b, c};
  return debug(Expression<AffineNodeOp>(nodes, transA, transB, scalar), "test");
}

Expr plus(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr swish(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr tanh(const std::vector<Expr>& nodes) {
  return Expression<TanhNodeOp>(nodes);
}

Expr logit(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr relu(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr leakyrelu(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr prelu(const std::vector<Expr>&, float alpha) {
  ABORT("Not implemented");
}

Expr sqrt(Expr a, float eps) {
  return Expression<SqrtNodeOp>(a, eps);
}

Expr square(Expr a) {
  return Expression<SquareNodeOp>(a);
}

Expr layer_norm(Expr x,
                Expr gamma,
                Expr beta /*= nullptr*/,
                float eps /*= 1e-9*/) {
  std::vector<Expr> nodes = {x, gamma};
  if(beta)
    nodes.push_back(beta);
  return Expression<LayerNormalizationOp>(nodes, eps);
}

Expr highway(Expr y, Expr x, Expr t) {
  std::vector<Expr> nodes = {y, x, t};
  return Expression<HighwayNodeOp>(nodes);
}

Expr highway(const std::string prefix, Expr x) {
  // clang-format off
  size_t outDim = x->shape()[-1];
  auto g = mlp::dense(x->graph())
      ("prefix", prefix + "_highway_d1")
      ("dim", outDim)
      ("activation", mlp::act::logit)
      .construct()->apply(x);
  auto relued = mlp::dense(x->graph())
      ("prefix", prefix + "_highway_d2")
      ("dim", outDim)
      ("activation", mlp::act::ReLU)
      .construct()->apply(x);
  return (g * relued) + ((1 - g) * x);
  // clang-format on
}

// Expr batch_norm(Expr x, Expr gamma, Expr beta) {
//  auto mju = mean(x, keywords::axis=0);
//  auto xmmju = x - mju;
//  auto std = sqrt(mean(square(xmmju), keywords::axis=0), 1e-9);
//
//  if(beta)
//    return gamma * (xmmju / std) + beta;
//  else
//    return gamma * (xmmju / std);
//}

Expr shift(Expr a, Shape shift) {
  return Expression<ShiftNodeOp>(a, shift);
}

// Expr lexical_bias(Expr logits, Expr att, float eps, Ptr<sparse::CSR> lf) {
//  return Expression<LexicalProbNodeOp>(logits, att, eps, lf);
//}

#ifdef CUDA_FOUND

Expr avg_pooling(Expr x,
                 int height,
                 int width,
                 int padHeight,
                 int padWidth,
                 int strideHeight,
                 int strideWidth) {
  return Expression<PoolingOp>(
      x, height, width, padHeight, padWidth, strideHeight, strideWidth, "avg");
}

Expr max_pooling(Expr x,
                 int height,
                 int width,
                 int padHeight,
                 int padWidth,
                 int strideHeight,
                 int strideWidth) {
  return Expression<PoolingOp>(
      x, height, width, padHeight, padWidth, strideHeight, strideWidth, "max");
}

Expr convert2cudnnFormat(Expr x) {
  int numWords = x->shape()[0];
  int numExamples = x->shape()[1];
  int embSize = x->shape()[2];

  std::vector<size_t> newIndeces;
  for(int b = 0; b < numExamples; ++b) {
    for(int t = 0; t < numWords; ++t) {
      newIndeces.push_back((t * numExamples) + b);
    }
  }

  auto xRows = reshape(x, {x->shape()[0] * x->shape()[1], x->shape()[2]});

  Shape outShape({numExamples, 1, numWords, embSize});
  return reshape(rows(xRows, newIndeces), outShape);
}

Expr convertFromcudnnFormat(Expr x) {
  int batchDim = x->shape()[0];
  int sentenceDim = x->shape()[2];
  int embSize = x->shape()[3];

  auto reshapedX = reshape(x, {batchDim * sentenceDim, embSize});

  std::vector<size_t> newIndeces;
  for(int t = 0; t < sentenceDim; ++t) {
    for(int b = 0; b < batchDim; ++b) {
      newIndeces.push_back(b * sentenceDim + t);
    }
  }

  Shape shape({batchDim, sentenceDim, embSize});
  return reshape(rows(reshapedX, newIndeces), shape);
}

Expr pooling_with_masking(Expr x, Expr mask, int width, bool isEven) {
  return Expression<PoolingWithMaskingOp>(x, mask, width, isEven);
}

#endif
}
<commit_msg>remove debug message<commit_after>#include "graph/expression_operators.h"
#include "layers/constructors.h"

#include "graph/node_operators.h"
#include "graph/node_operators_binary.h"
#include "graph/node_operators_unary.h"

namespace marian {

Expr debug(Expr a, const std::string& message) {
  a->debug(message);
  return a;
}

Expr logit(Expr a) {
  return Expression<LogitNodeOp>(a);
}

Expr relu(Expr a) {
  return Expression<ReLUNodeOp>(a);
}

Expr leakyrelu(Expr a) {
  return Expression<PReLUNodeOp>(0.01f, a);
}

Expr prelu(Expr a, float alpha) {
  return Expression<PReLUNodeOp>(alpha, a);
}

Expr log(Expr a) {
  return Expression<LogNodeOp>(a);
};

Expr exp(Expr a) {
  return Expression<ExpNodeOp>(a);
};

Expr swish(Expr a) {
  return Expression<SwishNodeOp>(a);
}

Expr operator-(Expr a) {
  return Expression<NegNodeOp>(a);
};

Expr softmax(Expr a, Expr mask) {
  return Expression<SoftmaxNodeOp>(a, mask);
}

Expr logsoftmax(Expr a) {
  return Expression<LogSoftmaxNodeOp>(a);
}

/*********************************************************/

Expr operator+(Expr a, Expr b) {
  return Expression<PlusNodeOp>(a, b);
}

Expr operator-(Expr a, Expr b) {
  return Expression<MinusNodeOp>(a, b);
}

Expr operator*(Expr a, Expr b) {
  return Expression<MultNodeOp>(a, b);
}

Expr operator/(Expr a, Expr b) {
  return Expression<DivNodeOp>(a, b);
}

/*********************************************************/

Expr operator+(Expr a, float b) {
  return Expression<ScalarAddNodeOp>(a, b);
}

Expr operator+(float a, Expr b) {
  return Expression<ScalarAddNodeOp>(b, a);
}

Expr operator-(Expr a, float b) {
  return Expression<ScalarAddNodeOp>(a, -b);
}

Expr operator-(float a, Expr b) {
  return Expression<ScalarAddNodeOp>(-b, a);
}

Expr operator*(float a, Expr b) {
  return Expression<ScalarMultNodeOp>(b, a);
}

Expr operator*(Expr a, float b) {
  return Expression<ScalarMultNodeOp>(a, b);
}

Expr operator/(Expr a, float b) {
  return Expression<ScalarMultNodeOp>(a, 1.f / b);
}

// Expr pow(float a, Expr b) {
//  return Expression<Scalar1PowNodeOp>(a, b);
//
//}
//
// Expr pow(Expr a, float b) {
//  return Expression<Scalar2PowNodeOp>(a, b);
//
//}
//
// Expr pow(Expr a, Expr b) {
//  return Expression<PowNodeOp>(a, b);
//}

/*********************************************************/

Expr concatenate(const std::vector<Expr>& concats, keywords::axis_k ax) {
  return Expression<ConcatenateNodeOp>(concats, ax);
}

Expr repeat(Expr a, size_t repeats, keywords::axis_k ax) {
  if(repeats == 1)
    return a;
  return concatenate(std::vector<Expr>(repeats, a), ax);
}

Expr reshape(Expr a, Shape shape) {
  return Expression<ReshapeNodeOp>(a, shape);
}

Expr atleast_1d(Expr a) {
  return atleast_nd(a, 1);
}

Expr atleast_2d(Expr a) {
  return atleast_nd(a, 2);
}

Expr atleast_3d(Expr a) {
  return atleast_nd(a, 3);
}

Expr atleast_4d(Expr a) {
  return atleast_nd(a, 4);
}

Expr atleast_nd(Expr a, size_t dims) {
  if(a->shape().size() >= dims)
    return a;

  Shape nShape;
  nShape.resize(dims);
  for(int i = 1; i <= a->shape().size(); ++i)
    nShape.set(-i, a->shape()[-i]);

  return reshape(a, nShape);
}

Expr flatten(Expr a) {
  Shape shape = {a->shape().elements()};
  return Expression<ReshapeNodeOp>(a, shape);
}

Expr flatten_2d(Expr a) {
  Shape shape = {a->shape().elements() / a->shape()[-1], a->shape()[-1]};

  return Expression<ReshapeNodeOp>(a, shape);
}

Expr rows(Expr a, const std::vector<size_t>& indices) {
  return Expression<RowsNodeOp>(a, indices);
}

Expr cols(Expr a, const std::vector<size_t>& indices) {
  return Expression<ColsNodeOp>(a, indices);
}

Expr select(Expr a, int axis, const std::vector<size_t>& indices) {
  return Expression<SelectNodeOp>(a, axis, indices);
}

Expr sum(Expr a, keywords::axis_k ax) {
  return Expression<SumNodeOp>(a, ax);
}

Expr mean(Expr a, keywords::axis_k ax) {
  return Expression<MeanNodeOp>(a, ax);
}

Expr scalar_product(Expr a, Expr b, keywords::axis_k ax) {
  return Expression<ScalarProductNodeOp>(a, b, ax);
}

Expr weighted_average(Expr in, Expr weights, keywords::axis_k ax) {
  auto p = scalar_product(in, weights, ax);
  auto s = sum(weights, ax);
  return p / s;
}

Expr dot(Expr a, Expr b, bool transA, bool transB, float scalar) {
  return Expression<DotNodeOp>(a, b, transA, transB, scalar);
}

Expr bdot(Expr a, Expr b, bool transA, bool transB, float scalar) {
  return Expression<DotBatchedNodeOp>(a, b, transA, transB, scalar);
}

Expr transpose(Expr a) {
  std::vector<int> axes(a->shape().size());
  for(int i = 0; i < axes.size(); ++i) {
    axes[i] = i;
  }
  if(axes.size() > 1) {
    axes[axes.size() - 1] = axes.size() - 2;
    axes[axes.size() - 2] = axes.size() - 1;
  }
  return Expression<TransposeNodeOp>(a, axes);
}

Expr transpose(Expr a, const std::vector<int>& axes) {
  return Expression<TransposeNodeOp>(a, axes);
}

Expr step(Expr a, int step, int axis) {
  return Expression<StepNodeOp>(a, step, axis);
}

Expr cross_entropy(Expr a, Expr b) {
  // auto sOrig = a->shape();
  // auto sOut = a->shape();
  // Shape sTemp({sOrig[0] * sOrig[2] * sOrig[3], sOrig[1], 1, 1});
  // sOut.set(1, 1);
  // return reshape(Expression<CrossEntropyNodeOp>(reshape(a, sTemp), b), sOut);

  return Expression<CrossEntropyNodeOp>(a, b);
}

Expr affine(Expr a, Expr b, Expr c, bool transA, bool transB, float scalar) {
  std::vector<Expr> nodes = {a, b, c};
  return Expression<AffineNodeOp>(nodes, transA, transB, scalar);
}

Expr plus(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr swish(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr tanh(const std::vector<Expr>& nodes) {
  return Expression<TanhNodeOp>(nodes);
}

Expr logit(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr relu(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr leakyrelu(const std::vector<Expr>&) {
  ABORT("Not implemented");
}

Expr prelu(const std::vector<Expr>&, float alpha) {
  ABORT("Not implemented");
}

Expr sqrt(Expr a, float eps) {
  return Expression<SqrtNodeOp>(a, eps);
}

Expr square(Expr a) {
  return Expression<SquareNodeOp>(a);
}

Expr layer_norm(Expr x,
                Expr gamma,
                Expr beta /*= nullptr*/,
                float eps /*= 1e-9*/) {
  std::vector<Expr> nodes = {x, gamma};
  if(beta)
    nodes.push_back(beta);
  return Expression<LayerNormalizationOp>(nodes, eps);
}

Expr highway(Expr y, Expr x, Expr t) {
  std::vector<Expr> nodes = {y, x, t};
  return Expression<HighwayNodeOp>(nodes);
}

Expr highway(const std::string prefix, Expr x) {
  // clang-format off
  size_t outDim = x->shape()[-1];
  auto g = mlp::dense(x->graph())
      ("prefix", prefix + "_highway_d1")
      ("dim", outDim)
      ("activation", mlp::act::logit)
      .construct()->apply(x);
  auto relued = mlp::dense(x->graph())
      ("prefix", prefix + "_highway_d2")
      ("dim", outDim)
      ("activation", mlp::act::ReLU)
      .construct()->apply(x);
  return (g * relued) + ((1 - g) * x);
  // clang-format on
}

// Expr batch_norm(Expr x, Expr gamma, Expr beta) {
//  auto mju = mean(x, keywords::axis=0);
//  auto xmmju = x - mju;
//  auto std = sqrt(mean(square(xmmju), keywords::axis=0), 1e-9);
//
//  if(beta)
//    return gamma * (xmmju / std) + beta;
//  else
//    return gamma * (xmmju / std);
//}

Expr shift(Expr a, Shape shift) {
  return Expression<ShiftNodeOp>(a, shift);
}

// Expr lexical_bias(Expr logits, Expr att, float eps, Ptr<sparse::CSR> lf) {
//  return Expression<LexicalProbNodeOp>(logits, att, eps, lf);
//}

#ifdef CUDA_FOUND

Expr avg_pooling(Expr x,
                 int height,
                 int width,
                 int padHeight,
                 int padWidth,
                 int strideHeight,
                 int strideWidth) {
  return Expression<PoolingOp>(
      x, height, width, padHeight, padWidth, strideHeight, strideWidth, "avg");
}

Expr max_pooling(Expr x,
                 int height,
                 int width,
                 int padHeight,
                 int padWidth,
                 int strideHeight,
                 int strideWidth) {
  return Expression<PoolingOp>(
      x, height, width, padHeight, padWidth, strideHeight, strideWidth, "max");
}

Expr convert2cudnnFormat(Expr x) {
  int numWords = x->shape()[0];
  int numExamples = x->shape()[1];
  int embSize = x->shape()[2];

  std::vector<size_t> newIndeces;
  for(int b = 0; b < numExamples; ++b) {
    for(int t = 0; t < numWords; ++t) {
      newIndeces.push_back((t * numExamples) + b);
    }
  }

  auto xRows = reshape(x, {x->shape()[0] * x->shape()[1], x->shape()[2]});

  Shape outShape({numExamples, 1, numWords, embSize});
  return reshape(rows(xRows, newIndeces), outShape);
}

Expr convertFromcudnnFormat(Expr x) {
  int batchDim = x->shape()[0];
  int sentenceDim = x->shape()[2];
  int embSize = x->shape()[3];

  auto reshapedX = reshape(x, {batchDim * sentenceDim, embSize});

  std::vector<size_t> newIndeces;
  for(int t = 0; t < sentenceDim; ++t) {
    for(int b = 0; b < batchDim; ++b) {
      newIndeces.push_back(b * sentenceDim + t);
    }
  }

  Shape shape({batchDim, sentenceDim, embSize});
  return reshape(rows(reshapedX, newIndeces), shape);
}

Expr pooling_with_masking(Expr x, Expr mask, int width, bool isEven) {
  return Expression<PoolingWithMaskingOp>(x, mask, width, isEven);
}

#endif
}
<|endoftext|>


<commit_before>/* -*- coding: utf-8; mode: c++; tab-width: 3; indent-tabs-mode: nil -*-

Copyright 2014, 2015
Raffaello D. Di Napoli

This file is part of Abaclade.

Abaclade is free software: you can redistribute it and/or modify it under the terms of the GNU
General Public License as published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

Abaclade is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
Public License for more details.

You should have received a copy of the GNU General Public License along with Abaclade. If not, see
<http://www.gnu.org/licenses/>.
--------------------------------------------------------------------------------------------------*/

#include <abaclade.hxx>
#include <abaclade/bitmanip.hxx>

#if ABC_HOST_API_POSIX
   #include <pthread.h>
#endif

////////////////////////////////////////////////////////////////////////////////////////////////////

namespace abc { namespace detail {

#if ABC_HOST_API_POSIX
   //! One-time initializer for g_pthkey.
   static pthread_once_t g_pthonce = PTHREAD_ONCE_INIT;
   //! TLS key.
   static pthread_key_t g_pthkey;
#elif ABC_HOST_API_WIN32
   //! TLS index.
   static DWORD g_iTls = TLS_OUT_OF_INDEXES;
#endif

ABC_COLLECTIONS_STATIC_LIST_DEFINE_SUBCLASS_STATIC_MEMBERS(thread_local_storage)
std::size_t thread_local_storage::sm_cb = 0;
std::size_t thread_local_storage::sm_cbFrozen = 0;

thread_local_storage::thread_local_storage() :
   m_pb(new std::int8_t[sm_cb]) {
   if (sm_cbFrozen == 0) {
      // Track the size of this first block.
      sm_cbFrozen = sm_cb;
   }

   /* Assign the TLS slot immediately, so that if any of the construct()s calls get() we won’t end
   up with an infinitely-recursive call. */
#if ABC_HOST_API_POSIX
   pthread_setspecific(g_pthkey, this);
#elif ABC_HOST_API_WIN32
   ::TlsSetValue(g_iTls, this);
#endif

   // Iterate over the list to construct TLS for this thread.
   for (auto it(begin()), itEnd(end()); it != itEnd; ++it) {
      it->construct(get_storage(it->m_ibStorageOffset));
   }
}

thread_local_storage::~thread_local_storage() {
   // Iterate backwards over the list to destruct TLS for this thread.
   for (auto it(rbegin()), itEnd(rend()); it != itEnd; ++it) {
      it->destruct(get_storage(it->m_ibStorageOffset));
   }
}

/*static*/ void thread_local_storage::add_var(thread_local_var_impl * ptlvi, std::size_t cb) {
   // Calculate the offset for *ptlvi’s storage and increase sm_cb accordingly.
   ptlvi->m_ibStorageOffset = sm_cb;
   sm_cb += bitmanip::ceiling_to_pow2_multiple(cb, sizeof(abc::max_align_t));
   if (sm_cbFrozen && sm_cb > sm_cbFrozen) {
      // TODO: can’t log/report anything since no thread locals are available! Fix me!
      std::abort();
   }
}

/*static*/ void thread_local_storage::alloc_slot() {
#if ABC_HOST_API_POSIX
   if (int iErr = pthread_key_create(&g_pthkey, &destruct)) {
      ABC_UNUSED_ARG(iErr);
      // throw an exception (iErr).
   }
#elif ABC_HOST_API_WIN32
   g_iTls = ::TlsAlloc();
   if (g_iTls == TLS_OUT_OF_INDEXES) {
      // throw an exception (::GetLastError()).
   }
#endif
}

#if ABC_HOST_API_POSIX
/*static*/ void thread_local_storage::destruct(void * pThis /*= get()*/) {
   delete static_cast<thread_local_storage *>(pThis);
}
#endif

#if ABC_HOST_API_WIN32
/*static*/ bool thread_local_storage::dllmain_hook(unsigned iReason) {
   if (iReason == DLL_PROCESS_ATTACH || iReason == DLL_THREAD_ATTACH) {
      if (iReason == DLL_PROCESS_ATTACH) {
         alloc_slot();
      }
      // Not calling construct() since initialization of TLS is lazy.
   } else if (iReason == DLL_THREAD_DETACH || iReason == DLL_PROCESS_DETACH) {
      /* Allow get() to return nullptr if the TLS slot was not initialized for this thread, in which
      case nothing will happen. */
      delete get(false);
      if (iReason == DLL_PROCESS_DETACH) {
         free_slot();
      }
   }
   // TODO: handle errors and return false in case.
   return true;
}
#endif //if ABC_HOST_API_WIN32

/*static*/ void thread_local_storage::free_slot() {
#if ABC_HOST_API_POSIX
   pthread_key_delete(g_pthkey);
#elif ABC_HOST_API_WIN32
   ::TlsFree(g_iTls);
#endif
}

/*static*/ thread_local_storage * thread_local_storage::get(bool bCreateNewIfNull /*= true*/) {
   void * pThis;
#if ABC_HOST_API_POSIX
   // With POSIX Threads we need a one-time call to alloc_slot().
   pthread_once(&g_pthonce, &alloc_slot);
   pThis = pthread_getspecific(g_pthkey);
#elif ABC_HOST_API_WIN32
   // Under Win32, alloc_slot() has already been called by dllmain_hook().
   pThis = ::TlsGetValue(g_iTls);
#endif
   if (pThis || !bCreateNewIfNull) {
      return static_cast<thread_local_storage *>(pThis);
   } else {
      // First call for this thread: initialize the TLS slot.
      return new thread_local_storage;
   }
}

}} //namespace abc::detail

////////////////////////////////////////////////////////////////////////////////////////////////////

namespace abc { namespace detail {

thread_local_var_impl::thread_local_var_impl(std::size_t cbObject) {
   // Initializes m_ibStorageOffset.
   thread_local_storage::add_var(this, cbObject);
}

}} //namespace abc::detail
<commit_msg>Add missing #include<commit_after>/* -*- coding: utf-8; mode: c++; tab-width: 3; indent-tabs-mode: nil -*-

Copyright 2014, 2015
Raffaello D. Di Napoli

This file is part of Abaclade.

Abaclade is free software: you can redistribute it and/or modify it under the terms of the GNU
General Public License as published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

Abaclade is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
Public License for more details.

You should have received a copy of the GNU General Public License along with Abaclade. If not, see
<http://www.gnu.org/licenses/>.
--------------------------------------------------------------------------------------------------*/

#include <abaclade.hxx>
#include <abaclade/bitmanip.hxx>

#include <cstdlib> // std::abort()

#if ABC_HOST_API_POSIX
   #include <pthread.h>
#endif

////////////////////////////////////////////////////////////////////////////////////////////////////

namespace abc { namespace detail {

#if ABC_HOST_API_POSIX
   //! One-time initializer for g_pthkey.
   static pthread_once_t g_pthonce = PTHREAD_ONCE_INIT;
   //! TLS key.
   static pthread_key_t g_pthkey;
#elif ABC_HOST_API_WIN32
   //! TLS index.
   static DWORD g_iTls = TLS_OUT_OF_INDEXES;
#endif

ABC_COLLECTIONS_STATIC_LIST_DEFINE_SUBCLASS_STATIC_MEMBERS(thread_local_storage)
std::size_t thread_local_storage::sm_cb = 0;
std::size_t thread_local_storage::sm_cbFrozen = 0;

thread_local_storage::thread_local_storage() :
   m_pb(new std::int8_t[sm_cb]) {
   if (sm_cbFrozen == 0) {
      // Track the size of this first block.
      sm_cbFrozen = sm_cb;
   }

   /* Assign the TLS slot immediately, so that if any of the construct()s calls get() we won’t end
   up with an infinitely-recursive call. */
#if ABC_HOST_API_POSIX
   pthread_setspecific(g_pthkey, this);
#elif ABC_HOST_API_WIN32
   ::TlsSetValue(g_iTls, this);
#endif

   // Iterate over the list to construct TLS for this thread.
   for (auto it(begin()), itEnd(end()); it != itEnd; ++it) {
      it->construct(get_storage(it->m_ibStorageOffset));
   }
}

thread_local_storage::~thread_local_storage() {
   // Iterate backwards over the list to destruct TLS for this thread.
   for (auto it(rbegin()), itEnd(rend()); it != itEnd; ++it) {
      it->destruct(get_storage(it->m_ibStorageOffset));
   }
}

/*static*/ void thread_local_storage::add_var(thread_local_var_impl * ptlvi, std::size_t cb) {
   // Calculate the offset for *ptlvi’s storage and increase sm_cb accordingly.
   ptlvi->m_ibStorageOffset = sm_cb;
   sm_cb += bitmanip::ceiling_to_pow2_multiple(cb, sizeof(abc::max_align_t));
   if (sm_cbFrozen && sm_cb > sm_cbFrozen) {
      // TODO: can’t log/report anything since no thread locals are available! Fix me!
      std::abort();
   }
}

/*static*/ void thread_local_storage::alloc_slot() {
#if ABC_HOST_API_POSIX
   if (int iErr = pthread_key_create(&g_pthkey, &destruct)) {
      ABC_UNUSED_ARG(iErr);
      // throw an exception (iErr).
   }
#elif ABC_HOST_API_WIN32
   g_iTls = ::TlsAlloc();
   if (g_iTls == TLS_OUT_OF_INDEXES) {
      // throw an exception (::GetLastError()).
   }
#endif
}

#if ABC_HOST_API_POSIX
/*static*/ void thread_local_storage::destruct(void * pThis /*= get()*/) {
   delete static_cast<thread_local_storage *>(pThis);
}
#endif

#if ABC_HOST_API_WIN32
/*static*/ bool thread_local_storage::dllmain_hook(unsigned iReason) {
   if (iReason == DLL_PROCESS_ATTACH || iReason == DLL_THREAD_ATTACH) {
      if (iReason == DLL_PROCESS_ATTACH) {
         alloc_slot();
      }
      // Not calling construct() since initialization of TLS is lazy.
   } else if (iReason == DLL_THREAD_DETACH || iReason == DLL_PROCESS_DETACH) {
      /* Allow get() to return nullptr if the TLS slot was not initialized for this thread, in which
      case nothing will happen. */
      delete get(false);
      if (iReason == DLL_PROCESS_DETACH) {
         free_slot();
      }
   }
   // TODO: handle errors and return false in case.
   return true;
}
#endif //if ABC_HOST_API_WIN32

/*static*/ void thread_local_storage::free_slot() {
#if ABC_HOST_API_POSIX
   pthread_key_delete(g_pthkey);
#elif ABC_HOST_API_WIN32
   ::TlsFree(g_iTls);
#endif
}

/*static*/ thread_local_storage * thread_local_storage::get(bool bCreateNewIfNull /*= true*/) {
   void * pThis;
#if ABC_HOST_API_POSIX
   // With POSIX Threads we need a one-time call to alloc_slot().
   pthread_once(&g_pthonce, &alloc_slot);
   pThis = pthread_getspecific(g_pthkey);
#elif ABC_HOST_API_WIN32
   // Under Win32, alloc_slot() has already been called by dllmain_hook().
   pThis = ::TlsGetValue(g_iTls);
#endif
   if (pThis || !bCreateNewIfNull) {
      return static_cast<thread_local_storage *>(pThis);
   } else {
      // First call for this thread: initialize the TLS slot.
      return new thread_local_storage;
   }
}

}} //namespace abc::detail

////////////////////////////////////////////////////////////////////////////////////////////////////

namespace abc { namespace detail {

thread_local_var_impl::thread_local_var_impl(std::size_t cbObject) {
   // Initializes m_ibStorageOffset.
   thread_local_storage::add_var(this, cbObject);
}

}} //namespace abc::detail
<|endoftext|>


<commit_before>/***************************************************************************
 *   Copyright (C) 2006 by FThauer FHammer   *
 *   f.thauer@web.de   *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************/
#include "chattools.h"
#include "session.h"
#include "configfile.h"
#include "gametablestylereader.h"
#include "gamelobbydialogimpl.h"
#include <iostream>

using namespace std;

ChatTools::ChatTools(QLineEdit* l, ConfigFile *c, ChatType ct, QTextBrowser *b, QStandardItemModel *m, gameLobbyDialogImpl *lo) : nickAutoCompletitionCounter(0), myLineEdit(l), myNickListModel(m), myNickStringList(NULL), myTextBrowser(b), myChatType(ct), myConfig(c), myNick(""), myLobby(lo)
{
    myNick = QString::fromUtf8(myConfig->readConfigString("MyName").c_str());
    ignoreList = myConfig->readConfigStringList("PlayerIgnoreList");

}

ChatTools::~ChatTools()
{
}

void ChatTools::sendMessage() {

    if(myLineEdit->text().size() && mySession) {
        fillChatLinesHistory(myLineEdit->text());
        if(myChatType == INGAME_CHAT) {
            mySession->sendGameChatMessage(myLineEdit->text().toUtf8().constData());
        }
        else {
            mySession->sendLobbyChatMessage(myLineEdit->text().toUtf8().constData());
        }
        myLineEdit->setText("");
    }
}

void ChatTools::receiveMessage(QString playerName, QString message) {

    if(myTextBrowser) {

        message = message.replace("<","<");
        message = message.replace(">",">");

        //refresh myNick if it was changed during runtime
        myNick = QString::fromUtf8(myConfig->readConfigString("MyName").c_str());

        QString tempMsg;

        if(INET_LOBBY_CHAT && playerName == "(chat bot)" && message.startsWith(myNick)) {

            tempMsg = QString(""+message+"");
            //play beep sound only in INET-lobby-chat
            if(myLobby->isVisible() && myConfig->readConfigInt("PlayLobbyChatNotification")) {
                myLobby->getMyW()->getMySDLPlayer()->playSound("lobbychatnotify",0);
            }
        }
        else if(message.contains(myNick, Qt::CaseInsensitive)) {

            switch (myChatType) {
            case INET_LOBBY_CHAT: {
                    tempMsg = QString("palette().link().color().name()+";\">"+message+"");
                    //play beep sound only in INET-lobby-chat
                    //						TODO dont play when message is from yourself
                    if(myLobby->isVisible() && myConfig->readConfigInt("PlayLobbyChatNotification")) {
                        myLobby->getMyW()->getMySDLPlayer()->playSound("lobbychatnotify",0);
                    }
                }
                break;
            case LAN_LOBBY_CHAT: tempMsg = QString(""+message+"");
                break;
            case INGAME_CHAT: tempMsg = QString("getChatTextNickNotifyColor()+";\">"+message+"");
                break;
            default: tempMsg = message;
            }
        }
        else if(playerName == myNick) {
            switch (myChatType) {
            case INET_LOBBY_CHAT: tempMsg = QString("palette().link().color().name()+";\">"+message+"");
                break;
            case LAN_LOBBY_CHAT: tempMsg = QString(""+message+"");
                break;
            case INGAME_CHAT: tempMsg = QString("getChatLogTextColor()+";\">"+message+"");
                break;
            default: tempMsg = message;
            }
        }
        else {
            switch (myChatType) {
            case INET_LOBBY_CHAT: tempMsg = QString("palette().text().color().name()+";\">"+message+"");
                break;
            case LAN_LOBBY_CHAT: tempMsg = QString(""+message+"");
                break;
            case INGAME_CHAT: tempMsg = QString("getChatLogTextColor()+";\">"+message+"");
                break;
            default: tempMsg = message;
            }

        }

        bool nickFoundOnIgnoreList = false;
        list<std::string>::iterator it1;
        for(it1=ignoreList.begin(); it1 != ignoreList.end(); it1++) {

            if(playerName == QString::fromUtf8(it1->c_str())) {
                nickFoundOnIgnoreList = true;
            }
        }

        if(!nickFoundOnIgnoreList) {
            myTextBrowser->append(playerName + ": " + tempMsg);
        }
    }
}

void ChatTools::clearChat() {

    if(myTextBrowser)
        myTextBrowser->clear();
}

void ChatTools::checkInputLength(QString string) {

    if(string.toUtf8().length() > 120) myLineEdit->setMaxLength(string.length());
}

void ChatTools::fillChatLinesHistory(QString fillString) {

    chatLinesHistory << fillString;
    if(chatLinesHistory.size() > 50) chatLinesHistory.removeFirst();

}

void ChatTools::showChatHistoryIndex(int index) {

    if(index <= chatLinesHistory.size()) {

        // 		cout << chatLinesHistory.size() << " : " <<  index << endl;
        if(index > 0)
            myLineEdit->setText(chatLinesHistory.at(chatLinesHistory.size()-(index)));
        else
            myLineEdit->setText("");
    }
}

void ChatTools::nickAutoCompletition() {

    QString myChatString = myLineEdit->text();
    QStringList myChatStringList = myChatString.split(" ");

    QStringList matchStringList;

    if(nickAutoCompletitionCounter == 0) {

        if(myNickListModel) {
            int it = 0;
            while (myNickListModel->item(it)) {
                QString text = myNickListModel->item(it, 0)->data(Qt::DisplayRole).toString();
                if(text.startsWith(myChatStringList.last(), Qt::CaseInsensitive) && myChatStringList.last() != "") {
                    matchStringList << text;
                }
                ++it;
            }
        }

        if(!myNickStringList.isEmpty()) {

            QStringListIterator it(myNickStringList);
            while (it.hasNext()) {
                QString next = it.next();
                if (next.startsWith(myChatStringList.last(), Qt::CaseInsensitive) && myChatStringList.last() != "")
                    matchStringList << next;
            }
        }
    }

    if(!matchStringList.isEmpty() || nickAutoCompletitionCounter > 0) {

        myChatStringList.removeLast();

        // 		cout << nickAutoCompletitionCounter << endl;

        if(nickAutoCompletitionCounter == 0) {
            //first one
            lastChatString = myChatStringList.join(" ");
            lastMatchStringList = matchStringList;
        }

        if(nickAutoCompletitionCounter == lastMatchStringList.size()) nickAutoCompletitionCounter = 0;

        // 		cout << nickAutoCompletitionCounter << "\n";

        if(lastChatString == "")
            myLineEdit->setText(lastMatchStringList.at(nickAutoCompletitionCounter)+": ");
        else
            myLineEdit->setText(lastChatString+" "+lastMatchStringList.at(nickAutoCompletitionCounter)+" ");

        nickAutoCompletitionCounter++;
    }
}

void ChatTools::setChatTextEdited() {

    nickAutoCompletitionCounter = 0;
}

void ChatTools::refreshIgnoreList()
{
    ignoreList = myConfig->readConfigStringList("PlayerIgnoreList");
}
<commit_msg>fixes<commit_after>/***************************************************************************
 *   Copyright (C) 2006 by FThauer FHammer   *
 *   f.thauer@web.de   *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************/
#include "chattools.h"
#include "session.h"
#include "configfile.h"
#include "gametablestylereader.h"
#include "gamelobbydialogimpl.h"
#include <iostream>

using namespace std;

ChatTools::ChatTools(QLineEdit* l, ConfigFile *c, ChatType ct, QTextBrowser *b, QStandardItemModel *m, gameLobbyDialogImpl *lo) : nickAutoCompletitionCounter(0), myLineEdit(l), myNickListModel(m), myNickStringList(NULL), myTextBrowser(b), myChatType(ct), myConfig(c), myNick(""), myLobby(lo)
{
    myNick = QString::fromUtf8(myConfig->readConfigString("MyName").c_str());
    ignoreList = myConfig->readConfigStringList("PlayerIgnoreList");

}

ChatTools::~ChatTools()
{
}

void ChatTools::sendMessage() {

    if(myLineEdit->text().size() && mySession) {
        fillChatLinesHistory(myLineEdit->text());
        if(myChatType == INGAME_CHAT) {
            mySession->sendGameChatMessage(myLineEdit->text().toUtf8().constData());
        }
        else {
            mySession->sendLobbyChatMessage(myLineEdit->text().toUtf8().constData());
        }
        myLineEdit->setText("");
    }
}

void ChatTools::receiveMessage(QString playerName, QString message) {

    if(myTextBrowser) {

        message = message.replace("<","<");
        message = message.replace(">",">");

        //refresh myNick if it was changed during runtime
        myNick = QString::fromUtf8(myConfig->readConfigString("MyName").c_str());

        QString tempMsg;

        if(myChatType == INET_LOBBY_CHAT && playerName == "(chat bot)" && message.startsWith(myNick)) {

            tempMsg = QString(""+message+"");
            //play beep sound only in INET-lobby-chat
            if(myLobby->isVisible() && myConfig->readConfigInt("PlayLobbyChatNotification")) {
                myLobby->getMyW()->getMySDLPlayer()->playSound("lobbychatnotify",0);
            }
        }
        else if(message.contains(myNick, Qt::CaseInsensitive)) {

            switch (myChatType) {
            case INET_LOBBY_CHAT: {
                    tempMsg = QString("palette().link().color().name()+";\">"+message+"");
                    //play beep sound only in INET-lobby-chat
                    //						TODO dont play when message is from yourself
                    if(myLobby->isVisible() && myConfig->readConfigInt("PlayLobbyChatNotification")) {
                        myLobby->getMyW()->getMySDLPlayer()->playSound("lobbychatnotify",0);
                    }
                }
                break;
            case LAN_LOBBY_CHAT: tempMsg = QString(""+message+"");
                break;
            case INGAME_CHAT: tempMsg = QString("getChatTextNickNotifyColor()+";\">"+message+"");
                break;
            default: tempMsg = message;
            }
        }
        else if(playerName == myNick) {
            switch (myChatType) {
            case INET_LOBBY_CHAT: tempMsg = QString("palette().link().color().name()+";\">"+message+"");
                break;
            case LAN_LOBBY_CHAT: tempMsg = QString(""+message+"");
                break;
            case INGAME_CHAT: tempMsg = QString("getChatLogTextColor()+";\">"+message+"");
                break;
            default: tempMsg = message;
            }
        }
        else {
            switch (myChatType) {
            case INET_LOBBY_CHAT: tempMsg = QString("palette().text().color().name()+";\">"+message+"");
                break;
            case LAN_LOBBY_CHAT: tempMsg = QString(""+message+"");
                break;
            case INGAME_CHAT: tempMsg = QString("getChatLogTextColor()+";\">"+message+"");
                break;
            default: tempMsg = message;
            }

        }

        bool nickFoundOnIgnoreList = false;
        list<std::string>::iterator it1;
        for(it1=ignoreList.begin(); it1 != ignoreList.end(); it1++) {

            if(playerName == QString::fromUtf8(it1->c_str())) {
                nickFoundOnIgnoreList = true;
            }
        }

        if(!nickFoundOnIgnoreList) {
            myTextBrowser->append(playerName + ": " + tempMsg);
        }
    }
}

void ChatTools::clearChat() {

    if(myTextBrowser)
        myTextBrowser->clear();
}

void ChatTools::checkInputLength(QString string) {

    if(string.toUtf8().length() > 120) myLineEdit->setMaxLength(string.length());
}

void ChatTools::fillChatLinesHistory(QString fillString) {

    chatLinesHistory << fillString;
    if(chatLinesHistory.size() > 50) chatLinesHistory.removeFirst();

}

void ChatTools::showChatHistoryIndex(int index) {

    if(index <= chatLinesHistory.size()) {

        // 		cout << chatLinesHistory.size() << " : " <<  index << endl;
        if(index > 0)
            myLineEdit->setText(chatLinesHistory.at(chatLinesHistory.size()-(index)));
        else
            myLineEdit->setText("");
    }
}

void ChatTools::nickAutoCompletition() {

    QString myChatString = myLineEdit->text();
    QStringList myChatStringList = myChatString.split(" ");

    QStringList matchStringList;

    if(nickAutoCompletitionCounter == 0) {

        if(myNickListModel) {
            int it = 0;
            while (myNickListModel->item(it)) {
                QString text = myNickListModel->item(it, 0)->data(Qt::DisplayRole).toString();
                if(text.startsWith(myChatStringList.last(), Qt::CaseInsensitive) && myChatStringList.last() != "") {
                    matchStringList << text;
                }
                ++it;
            }
        }

        if(!myNickStringList.isEmpty()) {

            QStringListIterator it(myNickStringList);
            while (it.hasNext()) {
                QString next = it.next();
                if (next.startsWith(myChatStringList.last(), Qt::CaseInsensitive) && myChatStringList.last() != "")
                    matchStringList << next;
            }
        }
    }

    if(!matchStringList.isEmpty() || nickAutoCompletitionCounter > 0) {

        myChatStringList.removeLast();

        // 		cout << nickAutoCompletitionCounter << endl;

        if(nickAutoCompletitionCounter == 0) {
            //first one
            lastChatString = myChatStringList.join(" ");
            lastMatchStringList = matchStringList;
        }

        if(nickAutoCompletitionCounter == lastMatchStringList.size()) nickAutoCompletitionCounter = 0;

        // 		cout << nickAutoCompletitionCounter << "\n";

        if(lastChatString == "")
            myLineEdit->setText(lastMatchStringList.at(nickAutoCompletitionCounter)+": ");
        else
            myLineEdit->setText(lastChatString+" "+lastMatchStringList.at(nickAutoCompletitionCounter)+" ");

        nickAutoCompletitionCounter++;
    }
}

void ChatTools::setChatTextEdited() {

    nickAutoCompletitionCounter = 0;
}

void ChatTools::refreshIgnoreList()
{
    ignoreList = myConfig->readConfigStringList("PlayerIgnoreList");
}
<|endoftext|>


<commit_before>#ifndef BASE_OBSTACLE_DETECTOR_H_
#define BASE_OBSTACLE_DETECTOR_H_

#include <vector>
#include <algorithm>

#include <pcl/visualization/cloud_viewer.h>

#include "lepp2/VideoObserver.hpp"
#include "lepp2/BaseSegmenter.hpp"
#include "lepp2/NoopSegmenter.hpp"
#include "lepp2/EuclideanPlaneSegmenter.hpp"
#include "lepp2/ObstacleAggregator.hpp"
#include "lepp2/ObjectApproximator.hpp"
#include "lepp2/MomentOfInertiaApproximator.hpp"

using namespace lepp;

#include "lepp2/debug/timer.hpp"

template<class PointT>
class BaseObstacleDetector : public lepp::VideoObserver<PointT> {
public:
  BaseObstacleDetector();
  virtual ~BaseObstacleDetector() {}

  /**
   * VideoObserver interface method implementation.
   */
  virtual void notifyNewFrame(
      int idx,
      const typename pcl::PointCloud<PointT>::ConstPtr& point_cloud);

  /**
   * Attaches a new ObstacleAggregator, which will be notified of newly detected
   * obstacles by this detector.
   */
  void attachObstacleAggregator(
      boost::shared_ptr<ObstacleAggregator> aggregator);
  /**
   * Returns the point cloud that the detector is currently working with.
   *
   * NOTE: Needed only for the legacy code that relies of pulling the point
   *       cloud from the detector, instead of getting it as a parameter.
   */
  virtual typename pcl::PointCloud<PointT>::ConstPtr getPointCloud() const {
    return cloud_;
  }

protected:
  /// Some convenience typedefs
  typedef pcl::PointCloud<PointT> PointCloud;
  typedef typename pcl::PointCloud<PointT>::ConstPtr PointCloudConstPtr;

  /**
   * Notifies any observers about newly detected obstacles.
   */
  void notifyObstacles(std::vector<ObjectModelPtr> const& models);

private:
  typename pcl::PointCloud<PointT>::ConstPtr cloud_;

  /**
   * Tracks all attached ObstacleAggregators that wish to be notified of newly
   * detected obstacles.
   */
  std::vector<boost::shared_ptr<ObstacleAggregator> > aggregators;

  boost::shared_ptr<BaseSegmenter<PointT> > segmenter_;
  boost::shared_ptr<ObjectApproximator<PointT> > approximator_;

  /**
   * Performs a new update of the obstacle approximations.
   * Triggered when the detector is notified of a new frame (i.e. point cloud).
   */
  void update();
};

template<class PointT>
BaseObstacleDetector<PointT>::BaseObstacleDetector()
    : approximator_(new MomentOfInertiaObjectApproximator<PointT>()),
      segmenter_(new EuclideanPlaneSegmenter<PointT>()) {
  // TODO Allow for dependency injection.
}

template<class PointT>
void BaseObstacleDetector<PointT>::notifyNewFrame(
    int id,
    const typename pcl::PointCloud<PointT>::ConstPtr& point_cloud) {
  cloud_ = point_cloud;
  update();
}

template<class PointT>
void BaseObstacleDetector<PointT>::update() {
  Timer t;
  t.start();
  std::vector<PointCloudConstPtr> segments(segmenter_->segment(cloud_));

  // Iteratively approximate the segments
  size_t segment_count = segments.size();
  std::vector<ObjectModelPtr> models;
  for (size_t i = 0; i < segment_count; ++i) {
    std::vector<ObjectModelPtr> approximations =
      approximator_->approximate(segments[i]);
    // Add all approximations to the final return value.
    // For now the fact that multiple approximations were a part of one group
    // is ignored to keep compatibility with the old iterfaces.
    std::copy(approximations.begin(), approximations.end(),
              std::back_inserter(models));
  }
  t.stop();
  std::cerr << "Obstacle detection took " << t.duration() << std::endl;

  notifyObstacles(models);
}

template<class PointT>
void BaseObstacleDetector<PointT>::attachObstacleAggregator(
    boost::shared_ptr<ObstacleAggregator> aggregator) {
  aggregators.push_back(aggregator);
}

template<class PointT>
void BaseObstacleDetector<PointT>::notifyObstacles(
  std::vector<ObjectModelPtr> const& models) {
  size_t sz = aggregators.size();
  for (size_t i = 0; i < sz; ++i) {
    aggregators[i]->updateObstacles(models);
  }
}

#endif
<commit_msg>Wrap obstacle detection in a try-catch block<commit_after>#ifndef BASE_OBSTACLE_DETECTOR_H_
#define BASE_OBSTACLE_DETECTOR_H_

#include <vector>
#include <algorithm>

#include <pcl/visualization/cloud_viewer.h>

#include "lepp2/VideoObserver.hpp"
#include "lepp2/BaseSegmenter.hpp"
#include "lepp2/NoopSegmenter.hpp"
#include "lepp2/EuclideanPlaneSegmenter.hpp"
#include "lepp2/ObstacleAggregator.hpp"
#include "lepp2/ObjectApproximator.hpp"
#include "lepp2/MomentOfInertiaApproximator.hpp"

using namespace lepp;

#include "lepp2/debug/timer.hpp"

template<class PointT>
class BaseObstacleDetector : public lepp::VideoObserver<PointT> {
public:
  BaseObstacleDetector();
  virtual ~BaseObstacleDetector() {}

  /**
   * VideoObserver interface method implementation.
   */
  virtual void notifyNewFrame(
      int idx,
      const typename pcl::PointCloud<PointT>::ConstPtr& point_cloud);

  /**
   * Attaches a new ObstacleAggregator, which will be notified of newly detected
   * obstacles by this detector.
   */
  void attachObstacleAggregator(
      boost::shared_ptr<ObstacleAggregator> aggregator);
  /**
   * Returns the point cloud that the detector is currently working with.
   *
   * NOTE: Needed only for the legacy code that relies of pulling the point
   *       cloud from the detector, instead of getting it as a parameter.
   */
  virtual typename pcl::PointCloud<PointT>::ConstPtr getPointCloud() const {
    return cloud_;
  }

protected:
  /// Some convenience typedefs
  typedef pcl::PointCloud<PointT> PointCloud;
  typedef typename pcl::PointCloud<PointT>::ConstPtr PointCloudConstPtr;

  /**
   * Notifies any observers about newly detected obstacles.
   */
  void notifyObstacles(std::vector<ObjectModelPtr> const& models);

private:
  typename pcl::PointCloud<PointT>::ConstPtr cloud_;

  /**
   * Tracks all attached ObstacleAggregators that wish to be notified of newly
   * detected obstacles.
   */
  std::vector<boost::shared_ptr<ObstacleAggregator> > aggregators;

  boost::shared_ptr<BaseSegmenter<PointT> > segmenter_;
  boost::shared_ptr<ObjectApproximator<PointT> > approximator_;

  /**
   * Performs a new update of the obstacle approximations.
   * Triggered when the detector is notified of a new frame (i.e. point cloud).
   */
  void update();
};

template<class PointT>
BaseObstacleDetector<PointT>::BaseObstacleDetector()
    : approximator_(new MomentOfInertiaObjectApproximator<PointT>()),
      segmenter_(new EuclideanPlaneSegmenter<PointT>()) {
  // TODO Allow for dependency injection.
}

template<class PointT>
void BaseObstacleDetector<PointT>::notifyNewFrame(
    int id,
    const typename pcl::PointCloud<PointT>::ConstPtr& point_cloud) {
  cloud_ = point_cloud;
  try {
    update();
  } catch (...) {
    std::cerr << "ObstacleDetector: Obstacle detection failed ..." << std::endl;
  }
}

template<class PointT>
void BaseObstacleDetector<PointT>::update() {
  Timer t;
  t.start();
  std::vector<PointCloudConstPtr> segments(segmenter_->segment(cloud_));

  // Iteratively approximate the segments
  size_t segment_count = segments.size();
  std::vector<ObjectModelPtr> models;
  for (size_t i = 0; i < segment_count; ++i) {
    std::vector<ObjectModelPtr> approximations =
      approximator_->approximate(segments[i]);
    // Add all approximations to the final return value.
    // For now the fact that multiple approximations were a part of one group
    // is ignored to keep compatibility with the old iterfaces.
    std::copy(approximations.begin(), approximations.end(),
              std::back_inserter(models));
  }
  t.stop();
  std::cerr << "Obstacle detection took " << t.duration() << std::endl;

  notifyObstacles(models);
}

template<class PointT>
void BaseObstacleDetector<PointT>::attachObstacleAggregator(
    boost::shared_ptr<ObstacleAggregator> aggregator) {
  aggregators.push_back(aggregator);
}

template<class PointT>
void BaseObstacleDetector<PointT>::notifyObstacles(
  std::vector<ObjectModelPtr> const& models) {
  size_t sz = aggregators.size();
  for (size_t i = 0; i < sz; ++i) {
    aggregators[i]->updateObstacles(models);
  }
}

#endif
<|endoftext|>