<commit_before>#pragma once

#include <cassert>
#include <map>
#include <omp.h>

#include "global_config.hpp"

namespace mocc {
    /**
     * The \ref Timer class provides functionality for measuring the amount of
     * runtime spent on various tasks. Each \ref Timer can have a number of
     * "children," which comprise sub-\ref Timer for individual tasks of
     * interest.
     *
     * Every \ref Timer maintains a total elapsed time, which may be accessed
     * via the \ref Timer::time() method. A \ref Timer can be thought of as a
     * stopwatch that is started with the \ref Timer::tic() method, and stopped
     * with the \ref Timer::toc() method. The elapsed time is a sum of all time
     * spent between calls the \ref Timer::tic() and \ref Timer::toc().
     *
     * There is a global \ref RootTimer, which is treated as the parent \ref
     * Timer for the entire executable.
     */
    class Timer {
    public:
        /**
         * \brief Create a new \ref Timer object.
         */
        Timer(std::string name);

        /**
         * \brief Create a new \ref Timer object and possibly start it.
         *
         * \param name the name of the \ref Timer
         * \param start whether or not to start the timer at construction-time.
         *
         * This will create a new \ref Timer and possibly start the \ref Timer
         * automatically if \p start is \c true. This is useful for
         * instrumenting constructor time for objects with lots of heavy lifting
         * to do in their initialization list. In such cases, starting the \ref
         * Timer in the body of the initializer will miss much of the time spent
         * constructing the objects in the initializer list. Placing a \ref
         * Timer object at the top of the initializer list will allow
         * measurement of this type of code.
         */
        Timer(std::string name, bool start );

        /**
         * \brief Start the \ref Timer
         *
         * This starts the \ref Timer "running," by logging the wall time at
         * which the \ref tic() function was called. The timer can then be
         * stopped with a call to \ref toc().
         */
        void tic();

        /**
         * \brief Stop the \ref Timer
         *
         * This stops the \ref Timer and returns the amount of time that elapsed
         * between the calls to \ref tic() and the \ref toc(). The running
         * sum of time for the \ref Timer is also incremented by this duration.
         */
        real_t toc();

        /**
         * \brief Return a reference the child Timer of the passed name
         */
        Timer &operator[]( const std::string &name ) {
            return children_.at(name);
        }

        /**
         * \brief Return a const reference the child Timer of the passed name
         */
        const Timer &operator[]( const std::string &name ) const {
            return children_.at(name);
        }

        /**
         * \brief Print the entire \ref Timer tree to the provided output
         * stream.
         */
        void print( std::ostream &os, int level=0 ) const;

        /**
         * \brief Create a new child \ref Timer and return a reference to it.
         */
        Timer &new_timer( const std::string &name ) {
            children_.emplace( name, Timer(name) );
            return children_.at(name);
        }

        /**
         * \brief Create and return a new child \ref Timer, possibly starting it
         * automatically
         */
        Timer &new_timer( const std::string &name, bool start ) {
            children_.emplace( name, Timer(name, start) );
            return children_.at(name);
        }

        friend std::ostream &operator<<( std::ostream &os,
                const Timer &timer );

    private:
        std::string name_;
        real_t time_;
        bool running_;
        real_t wtime_;
        std::map<std::string, Timer> children_;
    };

    extern Timer RootTimer;

}
<commit_msg>Fix docs, add time() to Timer<commit_after>#pragma once

#include <cassert>
#include <map>
#include <omp.h>

#include "global_config.hpp"

namespace mocc {
    /**
     * The \ref Timer class provides functionality for measuring the amount of
     * runtime spent on various tasks. Each \ref Timer can have a number of
     * "children," which comprise sub-\ref Timer for individual tasks of
     * interest.
     *
     * Every \ref Timer maintains a total elapsed time, which may be accessed
     * via the \ref Timer::time() method. A \ref Timer can be thought of as a
     * stopwatch that is started with the \ref Timer::tic() method, and stopped
     * with the \ref Timer::toc() method. The elapsed time is a sum of all time
     * spent between calls the \ref Timer::tic() and \ref Timer::toc().
     *
     * There is a global \ref RootTimer, which is treated as the parent \ref
     * Timer for the entire executable.
     */
    class Timer {
    public:
        /**
         * \brief Create a new \ref Timer object.
         */
        Timer(std::string name);

        /**
         * \brief Create a new \ref Timer object and possibly start it.
         *
         * \param name the name of the \ref Timer
         * \param start whether or not to start the timer at construction-time.
         *
         * This will create a new \ref Timer and possibly start the \ref Timer
         * automatically if \p start is \c true. This is useful for
         * instrumenting constructor time for objects with lots of heavy lifting
         * to do in their initialization list. In such cases, starting the \ref
         * Timer in the body of the initializer will miss much of the time spent
         * constructing the objects in the initializer list. Placing a \ref
         * Timer object at the top of the initializer list will allow
         * measurement of this type of code.
         */
        Timer(std::string name, bool start );

        /**
         * \brief Start the \ref Timer
         *
         * This starts the \ref Timer "running," by logging the wall time at
         * which the \ref tic() function was called. The timer can then be
         * stopped with a call to \ref toc().
         */
        void tic();

        /**
         * \brief Stop the \ref Timer
         *
         * This stops the \ref Timer and returns the amount of time that elapsed
         * between the calls to \ref tic() and the \ref toc(). The running
         * sum of time for the \ref Timer is also incremented by this duration.
         */
        real_t toc();

        /**
         * \brief Return the time accumulated so far for the timer.
         *
         * If the timer is currently running, this will not include time since
         * last call to \ref tic().
         */
        real_t time() const {
            return time_;
        }

        /**
         * \brief Return a reference the child Timer of the passed name
         */
        Timer &operator[]( const std::string &name ) {
            return children_.at(name);
        }

        /**
         * \brief Return a const reference the child Timer of the passed name
         */
        const Timer &operator[]( const std::string &name ) const {
            return children_.at(name);
        }

        /**
         * \brief Print the entire \ref Timer tree to the provided output
         * stream.
         */
        void print( std::ostream &os, int level=0 ) const;

        /**
         * \brief Create a new child \ref Timer and return a reference to it.
         */
        Timer &new_timer( const std::string &name ) {
            children_.emplace( name, Timer(name) );
            return children_.at(name);
        }

        /**
         * \brief Create and return a new child \ref Timer, possibly starting it
         * automatically
         */
        Timer &new_timer( const std::string &name, bool start ) {
            children_.emplace( name, Timer(name, start) );
            return children_.at(name);
        }

        friend std::ostream &operator<<( std::ostream &os,
                const Timer &timer );

    private:
        std::string name_;
        real_t time_;
        bool running_;
        real_t wtime_;
        std::map<std::string, Timer> children_;
    };

    extern Timer RootTimer;

}
<|endoftext|>


<commit_before>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/profiler/convert/xplane_to_step_events.h"

#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/profiler/utils/metadata_matcher.h"
#include "tensorflow/core/profiler/utils/trace_utils.h"
#include "tensorflow/core/profiler/utils/xplane_schema.h"

namespace tensorflow {
namespace profiler {
namespace {

// Returns true if the given event_name is a step marker.
inline bool IsStepMarker(absl::string_view event_name) {
  return (str_util::StartsWith(event_name, "train") ||
          str_util::StartsWith(event_name, "test") ||
          str_util::StartsWith(event_name, "TraceContext")) &&
         !str_util::StrContains(event_name, "/");
}

// Returns true if the given event_name should be considered as real computation
// on CPU.
inline bool IsRealCpuCompute(absl::string_view event_name) {
  bool not_real = str_util::StartsWith(event_name, "EagerExecute") ||
                  str_util::StartsWith(event_name, "EagerLocalExecute") ||
                  str_util::StartsWith(event_name, "EagerKernelExecute") ||
                  str_util::StartsWith(event_name, "FunctionRun") ||
                  IsStepMarker(event_name);
  return !not_real;
}

}  // namespace

StepEvents ConvertHostThreadsXLineToStepEvents(
    const XLineVisitor& line, bool use_device_step_events,
    const StepEvents& device_step_events) {
  StepEvents result;
  line.ForEachEvent([&](const XEventVisitor& event) {
    int64 correlation_id = -1;
    int64 group_id = -1;
    event.ForEachStat([&](const XStatVisitor& stat) {
      if (stat.Type() == StatType::kCorrelationId) {
        correlation_id = stat.IntValue();
      } else if (stat.Type() == StatType::kGroupId) {
        group_id = stat.IntValue();
      }
    });
    if (group_id < 0) return;
    // Don't add CPU events when (1) it includes device step events and (2) it
    // doesn't have a device and that the group_id (i.e. step number) already
    // appears on the device. This will filter out all cpu events that do not
    // correspond to any steps executed on the device.
    if (use_device_step_events &&
        device_step_events.find(group_id) == device_step_events.end())
      return;
    Timespan timespan = Timespan(event.TimestampPs(), event.DurationPs());
    if (IsStepMarker(event.Name())) {
      result[group_id].AddMarker(
          StepMarker(/*device=*/false, event.Name(), timespan));
    } else if (IsRealCpuCompute(event.Name())) {
      EventTypeSpan event_type_span(
          ClassifyCpuEvent(event.Name(), correlation_id), timespan);
      result[group_id].AddEvent(event_type_span);
    }
  });
  return result;
}

StepEvents ConvertHostThreadsXPlaneToStepEvents(
    const XPlane& host_trace, bool use_device_step_events,
    const StepEvents& device_step_events) {
  StepEvents result;
  XPlaneVisitor plane(&host_trace);
  plane.ForEachLine([&](const XLineVisitor& line) {
    CombineStepEvents(ConvertHostThreadsXLineToStepEvents(
                          line, use_device_step_events, device_step_events),
                      &result);
  });
  return result;
}

StepEvents ConvertDeviceTraceXLineToStepEvents(const XLineVisitor& line) {
  int64 correlation_id = -1;
  int64 group_id = -1;
  StepEvents result;
  line.ForEachEvent([&](const XEventVisitor& event) {
    event.ForEachStat([&](const XStatVisitor& stat) {
      if (stat.Type() == StatType::kCorrelationId) {
        correlation_id = stat.IntValue();
      } else if (stat.Type() == StatType::kGroupId) {
        group_id = stat.IntValue();
      }
    });
    if (correlation_id >= 0 && group_id >= 0) {
      EventTypeSpan event_type_span(
          ClassifyGpuEvent(event.Name()),
          Timespan(event.TimestampPs(), event.DurationPs()));
      result[group_id].AddEvent(event_type_span);
    }
  });
  return result;
}

StepEvents ConvertDeviceTraceXPlaneToStepEvents(const XPlane& device_trace) {
  StepEvents result;
  XPlaneVisitor plane(&device_trace);
  plane.ForEachLine([&](const XLineVisitor& line) {
    if (IsDerivedThreadId(line.Id())) return;
    CombineStepEvents(ConvertDeviceTraceXLineToStepEvents(line), &result);
  });
  return result;
}

}  // namespace profiler
}  // namespace tensorflow
<commit_msg>Fix a bug that kernel launch events are over calculated.<commit_after>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/profiler/convert/xplane_to_step_events.h"

#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/profiler/utils/metadata_matcher.h"
#include "tensorflow/core/profiler/utils/trace_utils.h"
#include "tensorflow/core/profiler/utils/xplane_schema.h"

namespace tensorflow {
namespace profiler {
namespace {

// Returns true if the given event_name is a step marker.
inline bool IsStepMarker(absl::string_view event_name) {
  return (str_util::StartsWith(event_name, "train") ||
          str_util::StartsWith(event_name, "test") ||
          str_util::StartsWith(event_name, "TraceContext")) &&
         !str_util::StrContains(event_name, "/");
}

// Returns true if the given event_name should be considered as real computation
// on CPU.
inline bool IsRealCpuCompute(absl::string_view event_name) {
  bool not_real = str_util::StartsWith(event_name, "EagerExecute") ||
                  str_util::StartsWith(event_name, "EagerLocalExecute") ||
                  str_util::StartsWith(event_name, "EagerKernelExecute") ||
                  str_util::StartsWith(event_name, "FunctionRun") ||
                  IsStepMarker(event_name);
  return !not_real;
}

}  // namespace

StepEvents ConvertHostThreadsXLineToStepEvents(
    const XLineVisitor& line, bool use_device_step_events,
    const StepEvents& device_step_events) {
  StepEvents result;
  line.ForEachEvent([&](const XEventVisitor& event) {
    int64 correlation_id = -1;
    int64 group_id = -1;
    event.ForEachStat([&](const XStatVisitor& stat) {
      if (stat.Type() == StatType::kCorrelationId) {
        correlation_id = stat.IntValue();
      } else if (stat.Type() == StatType::kGroupId) {
        group_id = stat.IntValue();
      }
    });
    if (group_id < 0) return;
    // Don't add CPU events when (1) it includes device step events and (2) it
    // doesn't have a device and that the group_id (i.e. step number) already
    // appears on the device. This will filter out all cpu events that do not
    // correspond to any steps executed on the device.
    if (use_device_step_events &&
        device_step_events.find(group_id) == device_step_events.end())
      return;
    Timespan timespan = Timespan(event.TimestampPs(), event.DurationPs());
    if (IsStepMarker(event.Name())) {
      result[group_id].AddMarker(
          StepMarker(/*device=*/false, event.Name(), timespan));
    } else if (IsRealCpuCompute(event.Name())) {
      EventTypeSpan event_type_span(
          ClassifyCpuEvent(event.Name(), correlation_id), timespan);
      result[group_id].AddEvent(event_type_span);
    }
  });
  return result;
}

StepEvents ConvertHostThreadsXPlaneToStepEvents(
    const XPlane& host_trace, bool use_device_step_events,
    const StepEvents& device_step_events) {
  StepEvents result;
  XPlaneVisitor plane(&host_trace);
  plane.ForEachLine([&](const XLineVisitor& line) {
    CombineStepEvents(ConvertHostThreadsXLineToStepEvents(
                          line, use_device_step_events, device_step_events),
                      &result);
  });
  return result;
}

StepEvents ConvertDeviceTraceXLineToStepEvents(const XLineVisitor& line) {
  StepEvents result;
  line.ForEachEvent([&](const XEventVisitor& event) {
    int64 correlation_id = -1;
    int64 group_id = -1;
    event.ForEachStat([&](const XStatVisitor& stat) {
      if (stat.Type() == StatType::kCorrelationId) {
        correlation_id = stat.IntValue();
      } else if (stat.Type() == StatType::kGroupId) {
        group_id = stat.IntValue();
      }
    });
    if (correlation_id >= 0 && group_id >= 0) {
      EventTypeSpan event_type_span(
          ClassifyGpuEvent(event.Name()),
          Timespan(event.TimestampPs(), event.DurationPs()));
      result[group_id].AddEvent(event_type_span);
    }
  });
  return result;
}

StepEvents ConvertDeviceTraceXPlaneToStepEvents(const XPlane& device_trace) {
  StepEvents result;
  XPlaneVisitor plane(&device_trace);
  plane.ForEachLine([&](const XLineVisitor& line) {
    if (IsDerivedThreadId(line.Id())) return;
    CombineStepEvents(ConvertDeviceTraceXLineToStepEvents(line), &result);
  });
  return result;
}

}  // namespace profiler
}  // namespace tensorflow
<|endoftext|>


<commit_before>/*
 Copyright (c) 2018 Anon authors, see AUTHORS file.

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
*/

#include "aws_sqs.h"
#include <aws/sqs/model/QueueAttributeName.h>
#include <aws/sqs/model/SendMessageRequest.h>

using namespace Aws::SQS;
using json = nlohmann::json;

aws_sqs_listener::aws_sqs_listener(const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> &provider,
                                   const Aws::Client::ClientConfiguration &client_config,
                                   const Aws::String &queue_url,
                                   const std::function<bool(const Aws::SQS::Model::Message &m)> &handler,
                                   bool single_concurrent_message,
                                   size_t stack_size)
    : _client(provider, client_config),
      _queue_url(queue_url),
      _num_fibers(0),
      _exit_now(false),
      _process_msg(handler),
      _consecutive_errors(0),
      _single_concurrent_message(single_concurrent_message),
      _stack_size(stack_size),
      _continue_after_timeout([]{return true;})
{
}

aws_sqs_listener::aws_sqs_listener(const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> &provider,
                                   const Aws::Client::ClientConfiguration &client_config,
                                   const Aws::String &queue_url,
                                   const std::function<bool(const Aws::SQS::Model::Message &m, const std::function<void(bool delete_it)>&)> &handler,
                                   bool single_concurrent_message,
                                   size_t stack_size)
    : _client(provider, client_config),
      _queue_url(queue_url),
      _num_fibers(0),
      _exit_now(false),
      _process_msg_del(handler),
      _consecutive_errors(0),
      _single_concurrent_message(single_concurrent_message),
      _stack_size(stack_size),
      _continue_after_timeout([]{return true;})
{
}

void aws_sqs_listener::start()
{
  std::weak_ptr<aws_sqs_listener> wp = shared_from_this();

  fiber::run_in_fiber(
      [wp] {
        auto ths = wp.lock();
        if (ths)
          ths->start_listen();
      },
      aws_sqs_listener::_simple_stack_size, "aws_sqs_listener::aws_sqs_listener, start_listen");

  #if defined(ANON_DEBUG_TIMERS)
  anon_log("io_dispatch::schedule_task, aws_sqs_listener::start");
  #endif
  _timer_task = io_dispatch::schedule_task(
      [wp] {
        fiber::run_in_fiber(
            [wp] {
              auto ths = wp.lock();
              if (ths)
                ths->set_visibility_timeout();
            },
            aws_sqs_listener::_simple_stack_size, "aws_sqs_listener, set_visibility_timeout sweeper");
      },
      cur_time() + visibility_sweep_time);
}

aws_sqs_listener::~aws_sqs_listener()
{
  {
    _exit_now = true;
    fiber_lock l(_mtx);
    while (_num_fibers > 0)
      _num_fibers_cond.wait(l);
  }
  io_dispatch::remove_task(_timer_task);
}

std::function<bool(const Aws::SQS::Model::Message &m)> aws_sqs_listener::js_wrap(const std::function<bool(const Aws::SQS::Model::Message &m, const nlohmann::json &body)> &fn)
{
  return [fn](const Aws::SQS::Model::Message &m) -> bool {
    std::string body = get_body(m);
    try
    {
      json body_js = json::parse(body.begin(), body.end());
      try
      {
        return fn(m, body_js);
      }
      catch (const inval_message& exc)
      {
        anon_log_error("caught exception processing message: " << exc.what() << ", message body: '" << body << "'");
        return true;
      }
      catch (const std::exception &exc)
      {
        anon_log_error("caught exception processing message: " << exc.what() << ", message body: '" << body << "'");
        return false;
      }
      catch (...)
      {
        anon_log_error("caught unknown exception processing message, message body: '" << body << "'");
        return false;
      }
    }
    catch (const std::exception &exc)
    {
      anon_log_error("caught exception parsing message: " << exc.what() << ", message body: '" << body << "'");
      return true;
    }
    catch (...)
    {
      anon_log_error("caught unknown exception parsing message, message body: '" << body << "'");
      return true;
    }
  };
}

std::function<bool(const Aws::SQS::Model::Message &m, const std::function<void(bool delete_it)>& del)> aws_sqs_listener::js_wrap(const std::function<bool(const Aws::SQS::Model::Message &m, const std::function<void(bool delete_it)>& del, const nlohmann::json &body)> &fn)
{
  return [fn](const Aws::SQS::Model::Message &m, const std::function<void(bool delete_it)>& del) -> bool {
    std::string body = get_body(m);
    try
    {
      json body_js = json::parse(body.begin(), body.end());
      try
      {
        return fn(m, del, body_js);
      }
      catch (const inval_message& exc)
      {
        anon_log_error("caught exception processing message: " << exc.what() << ", message body: '" << body << "'");
        del(true);
        return true;
      }
      catch (const std::exception &exc)
      {
        anon_log_error("caught exception processing message: " << exc.what() << ", message body: '" << body << "'");
        return false;
      }
      catch (...)
      {
        anon_log_error("caught unknown exception processing message, message body: '" << body << "'");
        return false;
      }
    }
    catch (const std::exception &exc)
    {
      anon_log_error("caught exception parsing message: " << exc.what() << ", message body: '" << body << "'");
      del(true);
      return true;
    }
    catch (...)
    {
      anon_log_error("caught unknown exception parsing message, message body: '" << body << "'");
      del(true);
      return true;
    }
  };
}

void aws_sqs_listener::start_listen()
{
  Model::ReceiveMessageRequest req;
  req.WithQueueUrl(_queue_url).WithMaxNumberOfMessages(_single_concurrent_message ? 1 : max_messages_per_read).WithWaitTimeSeconds(read_wait_time);
  Aws::Vector<Model::QueueAttributeName> att;
  att.push_back(Model::QueueAttributeName::All);
  req.WithAttributeNames(std::move(att));
  std::weak_ptr<aws_sqs_listener> wp = shared_from_this();
  _client.ReceiveMessageAsync(req, [wp](const SQSClient *, const Model::ReceiveMessageRequest &, const Model::ReceiveMessageOutcome &out, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
    auto ths = wp.lock();
    if (!ths)
      return;
    fiber::rename_fiber("aws_sqs_listener::start_listen, ReceiveMessageAsync");
    if (!out.IsSuccess())
    {
      ++ths->_consecutive_errors;
      if (ths->_consecutive_errors > 10)
      {
        anon_log_error("SQS ReceiveMessage failed, _consecutive_errors: " << ths->_consecutive_errors << "\n"
                                                                          << out.GetError());
      }
      else
      {
        anon_log("SQS ReceiveMessage failed, _consecutive_errors: " << ths->_consecutive_errors);
      }

      fiber::msleep(2000);
    }
    else
    {
      ths->_consecutive_errors = 0;
      auto &messages = out.GetResult().GetMessages();
      auto num_messages = messages.size();

      if (num_messages > 0)
      {
        ths->_num_fibers += num_messages;
        for (auto &m : messages)
          fiber::run_in_fiber(
              [wp, m] {
                auto ths = wp.lock();
                if (!ths)
                  return;
                anon_log("adding message " << m.GetMessageId() << " to keep_alive");
                ths->add_to_keep_alive(m);
                bool success;
                if (ths->_process_msg) {
                  success = ths->_process_msg(m);
                  if (success)
                    ths->delete_message(m);
                  if (ths->_single_concurrent_message && !ths->_exit_now)
                    ths->start_listen();
                }
                else
                  success = ths->_process_msg_del(m, [wp, m](bool delete_it) {
                    auto ths = wp.lock();
                    if (ths) {
                      if (delete_it)
                        ths->delete_message(m);
                      else
                        ths->remove_from_keep_alive(m, true);
                    }
                  });
                if (!success)
                  ths->remove_from_keep_alive(m, true);
              },
              ths->_stack_size, "aws_sqs_listener, process_msg");
      } else {
        if (!ths->_continue_after_timeout())
          ths->_exit_now = true;
      }
    }

    if (ths->_consecutive_errors < 1000)
    {
      if (!ths->_single_concurrent_message && !ths->_exit_now)
      {
        fiber_lock l(ths->_mtx);
        while (ths->_num_fibers >= max_in_flight_fibers)
          ths->_num_fibers_cond.wait(l);
        fiber::run_in_fiber(
            [wp] {
              auto ths = wp.lock();
              if (ths)
                ths->start_listen();
            },
            aws_sqs_listener::_simple_stack_size, "aws_sqs_listener, restart_listen");
      }
    }
    else
      anon_log_error("too many consecutive errors, giving up...");
  });
}

void aws_sqs_listener::set_visibility_timeout()
{
  fiber_lock l(_mtx);
  auto numMessages = _alive_set.size();
  if (numMessages > 0)
  {
    Model::ChangeMessageVisibilityBatchRequest req;
    Aws::Vector<Aws::Vector<Model::ChangeMessageVisibilityBatchRequestEntry>> entries_v;
    int index = 0;
    for (auto it : _alive_set)
    {
      Model::ChangeMessageVisibilityBatchRequestEntry ent;
      std::ostringstream str;
      if (index % 10 == 0)
        entries_v.push_back(Aws::Vector<Model::ChangeMessageVisibilityBatchRequestEntry>());
      str << "message_" << ++index;
      ent.WithReceiptHandle(it.first).WithVisibilityTimeout(visibility_time).WithId(str.str());
      entries_v.back().push_back(ent);
    }
    for (auto &entries : entries_v)
    {
      req.WithQueueUrl(_queue_url).WithEntries(entries);
      auto nMessages = entries.size();
      _client.ChangeMessageVisibilityBatchAsync(req, [nMessages](const SQSClient *, const Model::ChangeMessageVisibilityBatchRequest &, const Model::ChangeMessageVisibilityBatchOutcome &out, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
        fiber::rename_fiber("aws_sqs_listener::set_visibility_timeout, ChangeMessageVisibilityBatchAsync");
        if (out.IsSuccess())
        {
          anon_log("batch visibilty reset for " << nMessages << " messages");
        }
        else
        {
          do_error("batch reset visibility for " << nMessages << " messages: " << out.GetError().GetMessage());
        }
      });
    }
  }

  // schedule the sweep to run again in visibility_sweep_time seconds
  std::weak_ptr<aws_sqs_listener> wp = shared_from_this();
  #if defined(ANON_DEBUG_TIMERS)
  anon_log("io_dispatch::schedule_task, aws_sqs_listener::set_visibility_timeout");
  #endif
  _timer_task = io_dispatch::schedule_task(
      [wp] {
        fiber::run_in_fiber(
            [wp] {
              auto ths = wp.lock();
              if (ths)
                ths->set_visibility_timeout();
            },
            aws_sqs_listener::_simple_stack_size, "aws_sqs_listener, set_visibility_timeout sweeper");
      },
      cur_time() + visibility_sweep_time);
}

void aws_sqs_listener::add_to_keep_alive(const Model::Message &m)
{
  fiber_lock l(_mtx);
  _alive_set[m.GetReceiptHandle()] = m.GetMessageId();
}

void aws_sqs_listener::remove_from_keep_alive(const Model::Message &m, bool reset_visibility)
{
  {
    fiber_lock l(_mtx);
    _alive_set.erase(m.GetReceiptHandle());
  }
  if (reset_visibility)
  {
    // encourage the system to try this message again soon by resetting its visility near 0
    Model::ChangeMessageVisibilityRequest req;
    req.WithQueueUrl(_queue_url).WithReceiptHandle(m.GetReceiptHandle()).WithVisibilityTimeout(visibility_immediate_retry_time);
    auto messageId = m.GetMessageId();
    _client.ChangeMessageVisibilityAsync(req, [messageId](const SQSClient *, const Model::ChangeMessageVisibilityRequest &, const Model::ChangeMessageVisibilityOutcome &out, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
      fiber::rename_fiber("aws_sqs_listener::remove_from_keep_alive, ChangeMessageVisibilityAsync");
      if (out.IsSuccess())
      {
        anon_log("reset message visibility near 0 for " << messageId);
      }
      else
      {
        do_error("reset message visibility near 0 for " << messageId << ", " << out.GetError().GetMessage());
      }
    });
  }
}

void aws_sqs_listener::delete_message(const Model::Message &m)
{
  Model::DeleteMessageRequest req;
  req.WithQueueUrl(_queue_url).WithReceiptHandle(m.GetReceiptHandle());
  auto messageId = m.GetMessageId();
  std::weak_ptr<aws_sqs_listener> wp = shared_from_this();
  _client.DeleteMessageAsync(req, [messageId, wp, m](const SQSClient *, const Model::DeleteMessageRequest &, const Model::DeleteMessageOutcome &out, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
    fiber::rename_fiber("aws_sqs_listener::delete_message, DeleteMessageAsync");
    if (out.IsSuccess())
    {
      anon_log("deleted SQS message " << messageId);
    }
    else
    {
      anon_log("delete SQS message failed, messageId:" << messageId << ", " << out.GetError().GetMessage());
    }
    auto ths = wp.lock();
    if (ths)
      ths->remove_from_keep_alive(m, false);
  });
}

aws_sqs_sender::aws_sqs_sender(const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> &provider,
                               const Aws::Client::ClientConfiguration &client_config,
                               const Aws::String &queue_url)
    : _client(provider, client_config),
      _queue_url(queue_url)
{
}

void aws_sqs_sender::send(const json &body,
                          const std::function<void(const bool success, const std::string &id, const std::string &errorReason)> &response)
{
  Model::SendMessageRequest req;
  req.WithQueueUrl(_queue_url).WithMessageBody(Aws::String(body.dump()));
  _client.SendMessageAsync(req, [response](const SQSClient *, const Model::SendMessageRequest &, const Model::SendMessageOutcome &outcome, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
    fiber::rename_fiber("aws_sqs_sender::send, SendMessageAsync");
    response(outcome.IsSuccess(),
             std::string(outcome.GetResult().GetMessageId()),
             std::string(outcome.GetError().GetMessage()));
  });
}
<commit_msg>less logging<commit_after>/*
 Copyright (c) 2018 Anon authors, see AUTHORS file.

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
*/

#include "aws_sqs.h"
#include <aws/sqs/model/QueueAttributeName.h>
#include <aws/sqs/model/SendMessageRequest.h>

using namespace Aws::SQS;
using json = nlohmann::json;

aws_sqs_listener::aws_sqs_listener(const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> &provider,
                                   const Aws::Client::ClientConfiguration &client_config,
                                   const Aws::String &queue_url,
                                   const std::function<bool(const Aws::SQS::Model::Message &m)> &handler,
                                   bool single_concurrent_message,
                                   size_t stack_size)
    : _client(provider, client_config),
      _queue_url(queue_url),
      _num_fibers(0),
      _exit_now(false),
      _process_msg(handler),
      _consecutive_errors(0),
      _single_concurrent_message(single_concurrent_message),
      _stack_size(stack_size),
      _continue_after_timeout([]{return true;})
{
}

aws_sqs_listener::aws_sqs_listener(const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> &provider,
                                   const Aws::Client::ClientConfiguration &client_config,
                                   const Aws::String &queue_url,
                                   const std::function<bool(const Aws::SQS::Model::Message &m, const std::function<void(bool delete_it)>&)> &handler,
                                   bool single_concurrent_message,
                                   size_t stack_size)
    : _client(provider, client_config),
      _queue_url(queue_url),
      _num_fibers(0),
      _exit_now(false),
      _process_msg_del(handler),
      _consecutive_errors(0),
      _single_concurrent_message(single_concurrent_message),
      _stack_size(stack_size),
      _continue_after_timeout([]{return true;})
{
}

void aws_sqs_listener::start()
{
  std::weak_ptr<aws_sqs_listener> wp = shared_from_this();

  fiber::run_in_fiber(
      [wp] {
        auto ths = wp.lock();
        if (ths)
          ths->start_listen();
      },
      aws_sqs_listener::_simple_stack_size, "aws_sqs_listener::aws_sqs_listener, start_listen");

  #if defined(ANON_DEBUG_TIMERS)
  anon_log("io_dispatch::schedule_task, aws_sqs_listener::start");
  #endif
  _timer_task = io_dispatch::schedule_task(
      [wp] {
        fiber::run_in_fiber(
            [wp] {
              auto ths = wp.lock();
              if (ths)
                ths->set_visibility_timeout();
            },
            aws_sqs_listener::_simple_stack_size, "aws_sqs_listener, set_visibility_timeout sweeper");
      },
      cur_time() + visibility_sweep_time);
}

aws_sqs_listener::~aws_sqs_listener()
{
  {
    _exit_now = true;
    fiber_lock l(_mtx);
    while (_num_fibers > 0)
      _num_fibers_cond.wait(l);
  }
  io_dispatch::remove_task(_timer_task);
}

std::function<bool(const Aws::SQS::Model::Message &m)> aws_sqs_listener::js_wrap(const std::function<bool(const Aws::SQS::Model::Message &m, const nlohmann::json &body)> &fn)
{
  return [fn](const Aws::SQS::Model::Message &m) -> bool {
    std::string body = get_body(m);
    try
    {
      json body_js = json::parse(body.begin(), body.end());
      try
      {
        return fn(m, body_js);
      }
      catch (const inval_message& exc)
      {
        anon_log_error("caught exception processing message: " << exc.what() << ", message body: '" << body << "'");
        return true;
      }
      catch (const std::exception &exc)
      {
        anon_log_error("caught exception processing message: " << exc.what() << ", message body: '" << body << "'");
        return false;
      }
      catch (...)
      {
        anon_log_error("caught unknown exception processing message, message body: '" << body << "'");
        return false;
      }
    }
    catch (const std::exception &exc)
    {
      anon_log_error("caught exception parsing message: " << exc.what() << ", message body: '" << body << "'");
      return true;
    }
    catch (...)
    {
      anon_log_error("caught unknown exception parsing message, message body: '" << body << "'");
      return true;
    }
  };
}

std::function<bool(const Aws::SQS::Model::Message &m, const std::function<void(bool delete_it)>& del)> aws_sqs_listener::js_wrap(const std::function<bool(const Aws::SQS::Model::Message &m, const std::function<void(bool delete_it)>& del, const nlohmann::json &body)> &fn)
{
  return [fn](const Aws::SQS::Model::Message &m, const std::function<void(bool delete_it)>& del) -> bool {
    std::string body = get_body(m);
    try
    {
      json body_js = json::parse(body.begin(), body.end());
      try
      {
        return fn(m, del, body_js);
      }
      catch (const inval_message& exc)
      {
        anon_log_error("caught exception processing message: " << exc.what() << ", message body: '" << body << "'");
        del(true);
        return true;
      }
      catch (const std::exception &exc)
      {
        anon_log_error("caught exception processing message: " << exc.what() << ", message body: '" << body << "'");
        return false;
      }
      catch (...)
      {
        anon_log_error("caught unknown exception processing message, message body: '" << body << "'");
        return false;
      }
    }
    catch (const std::exception &exc)
    {
      anon_log_error("caught exception parsing message: " << exc.what() << ", message body: '" << body << "'");
      del(true);
      return true;
    }
    catch (...)
    {
      anon_log_error("caught unknown exception parsing message, message body: '" << body << "'");
      del(true);
      return true;
    }
  };
}

void aws_sqs_listener::start_listen()
{
  Model::ReceiveMessageRequest req;
  req.WithQueueUrl(_queue_url).WithMaxNumberOfMessages(_single_concurrent_message ? 1 : max_messages_per_read).WithWaitTimeSeconds(read_wait_time);
  Aws::Vector<Model::QueueAttributeName> att;
  att.push_back(Model::QueueAttributeName::All);
  req.WithAttributeNames(std::move(att));
  std::weak_ptr<aws_sqs_listener> wp = shared_from_this();
  _client.ReceiveMessageAsync(req, [wp](const SQSClient *, const Model::ReceiveMessageRequest &, const Model::ReceiveMessageOutcome &out, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
    auto ths = wp.lock();
    if (!ths)
      return;
    fiber::rename_fiber("aws_sqs_listener::start_listen, ReceiveMessageAsync");
    if (!out.IsSuccess())
    {
      ++ths->_consecutive_errors;
      if (ths->_consecutive_errors > 10)
      {
        anon_log_error("SQS ReceiveMessage failed, _consecutive_errors: " << ths->_consecutive_errors << "\n"
                                                                          << out.GetError());
      }
      else
      {
        anon_log("SQS ReceiveMessage failed, _consecutive_errors: " << ths->_consecutive_errors);
      }

      fiber::msleep(2000);
    }
    else
    {
      ths->_consecutive_errors = 0;
      auto &messages = out.GetResult().GetMessages();
      auto num_messages = messages.size();

      if (num_messages > 0)
      {
        ths->_num_fibers += num_messages;
        for (auto &m : messages)
          fiber::run_in_fiber(
              [wp, m] {
                auto ths = wp.lock();
                if (!ths)
                  return;
                anon_log("adding message " << m.GetMessageId() << " to keep_alive");
                ths->add_to_keep_alive(m);
                bool success;
                if (ths->_process_msg) {
                  success = ths->_process_msg(m);
                  if (success)
                    ths->delete_message(m);
                  if (ths->_single_concurrent_message && !ths->_exit_now)
                    ths->start_listen();
                }
                else
                  success = ths->_process_msg_del(m, [wp, m](bool delete_it) {
                    auto ths = wp.lock();
                    if (ths) {
                      if (delete_it)
                        ths->delete_message(m);
                      else
                        ths->remove_from_keep_alive(m, true);
                    }
                  });
                if (!success)
                  ths->remove_from_keep_alive(m, true);
              },
              ths->_stack_size, "aws_sqs_listener, process_msg");
      } else {
        if (!ths->_continue_after_timeout())
          ths->_exit_now = true;
      }
    }

    if (ths->_consecutive_errors < 1000)
    {
      if (!ths->_single_concurrent_message && !ths->_exit_now)
      {
        fiber_lock l(ths->_mtx);
        while (ths->_num_fibers >= max_in_flight_fibers)
          ths->_num_fibers_cond.wait(l);
        fiber::run_in_fiber(
            [wp] {
              auto ths = wp.lock();
              if (ths)
                ths->start_listen();
            },
            aws_sqs_listener::_simple_stack_size, "aws_sqs_listener, restart_listen");
      }
    }
    else
      anon_log_error("too many consecutive errors, giving up...");
  });
}

void aws_sqs_listener::set_visibility_timeout()
{
  fiber_lock l(_mtx);
  auto numMessages = _alive_set.size();
  if (numMessages > 0)
  {
    Model::ChangeMessageVisibilityBatchRequest req;
    Aws::Vector<Aws::Vector<Model::ChangeMessageVisibilityBatchRequestEntry>> entries_v;
    int index = 0;
    for (auto it : _alive_set)
    {
      Model::ChangeMessageVisibilityBatchRequestEntry ent;
      std::ostringstream str;
      if (index % 10 == 0)
        entries_v.push_back(Aws::Vector<Model::ChangeMessageVisibilityBatchRequestEntry>());
      str << "message_" << ++index;
      ent.WithReceiptHandle(it.first).WithVisibilityTimeout(visibility_time).WithId(str.str());
      entries_v.back().push_back(ent);
    }
    for (auto &entries : entries_v)
    {
      req.WithQueueUrl(_queue_url).WithEntries(entries);
      auto nMessages = entries.size();
      _client.ChangeMessageVisibilityBatchAsync(req, [nMessages](const SQSClient *, const Model::ChangeMessageVisibilityBatchRequest &, const Model::ChangeMessageVisibilityBatchOutcome &out, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
        fiber::rename_fiber("aws_sqs_listener::set_visibility_timeout, ChangeMessageVisibilityBatchAsync");
        if (out.IsSuccess())
        {
          // anon_log("batch visibilty reset for " << nMessages << " message" << (nMessages > 1 ? "s" : ""));
        }
        else
        {
          do_error("batch reset visibility for " << nMessages << " messages: " << out.GetError().GetMessage());
        }
      });
    }
  }

  // schedule the sweep to run again in visibility_sweep_time seconds
  std::weak_ptr<aws_sqs_listener> wp = shared_from_this();
  #if defined(ANON_DEBUG_TIMERS)
  anon_log("io_dispatch::schedule_task, aws_sqs_listener::set_visibility_timeout");
  #endif
  _timer_task = io_dispatch::schedule_task(
      [wp] {
        fiber::run_in_fiber(
            [wp] {
              auto ths = wp.lock();
              if (ths)
                ths->set_visibility_timeout();
            },
            aws_sqs_listener::_simple_stack_size, "aws_sqs_listener, set_visibility_timeout sweeper");
      },
      cur_time() + visibility_sweep_time);
}

void aws_sqs_listener::add_to_keep_alive(const Model::Message &m)
{
  fiber_lock l(_mtx);
  _alive_set[m.GetReceiptHandle()] = m.GetMessageId();
}

void aws_sqs_listener::remove_from_keep_alive(const Model::Message &m, bool reset_visibility)
{
  {
    fiber_lock l(_mtx);
    _alive_set.erase(m.GetReceiptHandle());
  }
  if (reset_visibility)
  {
    // encourage the system to try this message again soon by resetting its visility near 0
    Model::ChangeMessageVisibilityRequest req;
    req.WithQueueUrl(_queue_url).WithReceiptHandle(m.GetReceiptHandle()).WithVisibilityTimeout(visibility_immediate_retry_time);
    auto messageId = m.GetMessageId();
    _client.ChangeMessageVisibilityAsync(req, [messageId](const SQSClient *, const Model::ChangeMessageVisibilityRequest &, const Model::ChangeMessageVisibilityOutcome &out, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
      fiber::rename_fiber("aws_sqs_listener::remove_from_keep_alive, ChangeMessageVisibilityAsync");
      if (out.IsSuccess())
      {
        anon_log("reset message visibility near 0 for " << messageId);
      }
      else
      {
        do_error("reset message visibility near 0 for " << messageId << ", " << out.GetError().GetMessage());
      }
    });
  }
}

void aws_sqs_listener::delete_message(const Model::Message &m)
{
  Model::DeleteMessageRequest req;
  req.WithQueueUrl(_queue_url).WithReceiptHandle(m.GetReceiptHandle());
  auto messageId = m.GetMessageId();
  std::weak_ptr<aws_sqs_listener> wp = shared_from_this();
  _client.DeleteMessageAsync(req, [messageId, wp, m](const SQSClient *, const Model::DeleteMessageRequest &, const Model::DeleteMessageOutcome &out, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
    fiber::rename_fiber("aws_sqs_listener::delete_message, DeleteMessageAsync");
    if (out.IsSuccess())
    {
      anon_log("deleted SQS message " << messageId);
    }
    else
    {
      anon_log("delete SQS message failed, messageId:" << messageId << ", " << out.GetError().GetMessage());
    }
    auto ths = wp.lock();
    if (ths)
      ths->remove_from_keep_alive(m, false);
  });
}

aws_sqs_sender::aws_sqs_sender(const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> &provider,
                               const Aws::Client::ClientConfiguration &client_config,
                               const Aws::String &queue_url)
    : _client(provider, client_config),
      _queue_url(queue_url)
{
}

void aws_sqs_sender::send(const json &body,
                          const std::function<void(const bool success, const std::string &id, const std::string &errorReason)> &response)
{
  Model::SendMessageRequest req;
  req.WithQueueUrl(_queue_url).WithMessageBody(Aws::String(body.dump()));
  _client.SendMessageAsync(req, [response](const SQSClient *, const Model::SendMessageRequest &, const Model::SendMessageOutcome &outcome, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
    fiber::rename_fiber("aws_sqs_sender::send, SendMessageAsync");
    response(outcome.IsSuccess(),
             std::string(outcome.GetResult().GetMessageId()),
             std::string(outcome.GetError().GetMessage()));
  });
}
<|endoftext|>


<commit_before>/*********************************************************************
 *
 * Copyright 2012 the original author or authors.
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *********************************************************************/
#include "resizer.h"

#include<ros/ros.h>
#include<sensor_msgs/CompressedImage.h>

extern "C" {
#include <libswscale/swscale.h>
#include <libavutil/mem.h>
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55,28,1)
#include <libavutil/frame.h>
#endif
}

#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1)
#define av_frame_alloc  avcodec_alloc_frame
#define av_frame_free   av_free
#endif

namespace rospilot {

using std::vector;
using sensor_msgs::CompressedImage;

bool Resizer::resizeInPlace(sensor_msgs::CompressedImage *image)
{
    if (originalWidth == targetWidth && originalHeight == targetHeight) {
        return true;
    }

    avpicture_fill(
            (AVPicture*) sourceFrame,
            image->data.data(),
            pixelFormat,
            originalWidth,
            originalHeight);

    sws_scale(
            context,
            sourceFrame->data,
            sourceFrame->linesize,
            0,
            originalHeight,
            outputFrame->data,
            outputFrame->linesize);

    int size = avpicture_layout(
            (AVPicture *) outputFrame,
            pixelFormat,
            targetWidth,
            targetHeight,
            outputBuffer,
            outputBufferSize);
    if (size != outputBufferSize) {
        ROS_ERROR("avpicture_layout failed: %d",size);
        return false;
    }

    image->data.clear();
    image->data.reserve(outputBufferSize);
    image->data.insert(image->data.begin(), outputBuffer, outputBuffer + outputBufferSize);

    return true;
}

Resizer::Resizer(
        int originalWidth,
        int originalHeight,
        int targetWidth,
        int targetHeight,
        PixelFormat pixelFormat)
{
    this->pixelFormat = pixelFormat;
    this->originalWidth = originalWidth;
    this->originalHeight = originalHeight;
    this->targetWidth = targetWidth;
    this->targetHeight = targetHeight;

    outputBufferSize = avpicture_get_size(pixelFormat, targetWidth, targetHeight);
    outputBuffer = new uint8_t[outputBufferSize];

    outputFrame = av_frame_alloc();
    avpicture_alloc((AVPicture*) outputFrame, pixelFormat, targetWidth, targetHeight);

    sourceFrame = av_frame_alloc();
    sourceFrame->width = originalWidth;
    sourceFrame->height = originalHeight;
    sourceFrame->format = pixelFormat;

    context = sws_getContext(
            originalWidth,
            originalHeight,
            pixelFormat,
            targetWidth,
            targetHeight,
            pixelFormat,
            SWS_BICUBIC,
            nullptr,
            nullptr,
            nullptr);
}

Resizer::~Resizer()
{
    av_frame_free(sourceFrame);
    av_frame_free(outputFrame);
    sws_freeContext(context);
    delete[] outputBuffer;
}

}
<commit_msg>Fix compilation error on vivid and utopic<commit_after>/*********************************************************************
 *
 * Copyright 2012 the original author or authors.
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *********************************************************************/
#include "resizer.h"

#include<ros/ros.h>
#include<sensor_msgs/CompressedImage.h>

extern "C" {
#include <libswscale/swscale.h>
#include <libavutil/mem.h>
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55,28,1)
#include <libavutil/frame.h>
#endif
}

#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1)
#define av_frame_alloc  avcodec_alloc_frame
#define av_frame_free   av_free
#endif

namespace rospilot {

using std::vector;
using sensor_msgs::CompressedImage;

bool Resizer::resizeInPlace(sensor_msgs::CompressedImage *image)
{
    if (originalWidth == targetWidth && originalHeight == targetHeight) {
        return true;
    }

    avpicture_fill(
            (AVPicture*) sourceFrame,
            image->data.data(),
            pixelFormat,
            originalWidth,
            originalHeight);

    sws_scale(
            context,
            sourceFrame->data,
            sourceFrame->linesize,
            0,
            originalHeight,
            outputFrame->data,
            outputFrame->linesize);

    int size = avpicture_layout(
            (AVPicture *) outputFrame,
            pixelFormat,
            targetWidth,
            targetHeight,
            outputBuffer,
            outputBufferSize);
    if (size != outputBufferSize) {
        ROS_ERROR("avpicture_layout failed: %d",size);
        return false;
    }

    image->data.clear();
    image->data.reserve(outputBufferSize);
    image->data.insert(image->data.begin(), outputBuffer, outputBuffer + outputBufferSize);

    return true;
}

Resizer::Resizer(
        int originalWidth,
        int originalHeight,
        int targetWidth,
        int targetHeight,
        PixelFormat pixelFormat)
{
    this->pixelFormat = pixelFormat;
    this->originalWidth = originalWidth;
    this->originalHeight = originalHeight;
    this->targetWidth = targetWidth;
    this->targetHeight = targetHeight;

    outputBufferSize = avpicture_get_size(pixelFormat, targetWidth, targetHeight);
    outputBuffer = new uint8_t[outputBufferSize];

    outputFrame = av_frame_alloc();
    avpicture_alloc((AVPicture*) outputFrame, pixelFormat, targetWidth, targetHeight);

    sourceFrame = av_frame_alloc();
    sourceFrame->width = originalWidth;
    sourceFrame->height = originalHeight;
    sourceFrame->format = pixelFormat;

    context = sws_getContext(
            originalWidth,
            originalHeight,
            pixelFormat,
            targetWidth,
            targetHeight,
            pixelFormat,
            SWS_BICUBIC,
            nullptr,
            nullptr,
            nullptr);
}

Resizer::~Resizer()
{
    av_frame_free(&sourceFrame);
    av_frame_free(&outputFrame);
    sws_freeContext(context);
    delete[] outputBuffer;
}

}
<|endoftext|>


<commit_before>/*
 * tripolar.cpp
 *
 * Created: 5/12/2015 4:02:05 PM
 *  Author: Alan
 */

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& USER SETUP
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
	*/

	#define F_CPU 16000000UL
	//#define DO_DEBUG
/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& INCLUDES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

#include <avr/io.h>
#include "bldcGimbal.h"

#include "afro_nfet.h"
#include "fets.h"

#include <util/delay.h>
#include "millis.h"
#include <stdlib.h>

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& FUNCTION PROTOTYPES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

void setup(void);
void loop(void);

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& STRUCTURES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

/*****************************************************************************************************/
/* STRUCT: servoIsrData_S																			 */
/** Data structure used to interact with the Servo Interrupt service routine						 */
/*****************************************************************************************************/
typedef struct servoIsrData_S
{

	volatile uint16_t startTimeStamp;
	volatile uint16_t stopTimeStamp;
	bool dataReady;
	bool waitRising; //If true, we are waiting for a rising edge, otherwise a falling

}servoIsrData_T;

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& VARIABLES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

bldcGimbal gimbal;
static servoIsrData_T servoIsrData;

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& INTERRUPT SERVICE ROUTINES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

ISR(TIMER1_CAPT_vect)
{
	sei();
	volatile uint16_t timeStamp;
	timeStamp = micros();
	if (servoIsrData.dataReady) return; //we already have a reading, so don't collect a new one till this one is read.

	if (servoIsrData.waitRising) {
		servoIsrData.startTimeStamp = timeStamp;  //If Rising Edge, record the start time
		TCCR1B &= ~_BV(ICES1);//Set interrupt for falling edge
		servoIsrData.waitRising = false;
	}
	else  //Otherwise its a falling edge so ..
	{
		servoIsrData.stopTimeStamp = timeStamp;	//Record the stop time..
		servoIsrData.dataReady = true;			//And signal that we have a value
		servoIsrData.waitRising = true;
		TCCR1B |= _BV(ICES1); //Set interrupt for rising edge
	}
}

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& SERVO ISR SUPPORT FUNCTIONS
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

inline void servo_begin(void)
{
	sei(); //Enable nested interrupts
	servoIsrData.dataReady = false;
	servoIsrData.waitRising = true;

	//Configure PCINT0 PIN (PB0) as an input
	DDRB &= ~_BV(DDB0);

	//DISABLE PULLUP RESISTOR (PB0)
	PORTB &= ~_BV(PORTB0);

	//Rising Edge Detect
	TCCR1B |= _BV(ICES1);

	//Enable Input Compare Interrupt
	TIMSK |= _BV(TICIE1);

}

inline bool servo_changed(void)
{
	if (servoIsrData.dataReady)
	{
		if (gimbal.motorPwm().icr1Conflict()) {
			TIMSK &= ~_BV(TICIE1);
			servoIsrData.dataReady = false;
			TIMSK |= _BV(TICIE1);
			redOn();
		}
	}
	return servoIsrData.dataReady;
}

inline uint16_t servo_value_uS(void)
{
	TIMSK &= ~_BV(TICIE1);
	uint16_t retVal = servoIsrData.stopTimeStamp - servoIsrData.startTimeStamp;
	TIMSK |= _BV(TICIE1);
	redOff();
	servoIsrData.dataReady = false;

	return retVal;
}

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& FUNCTIONS
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

int main(void)
{
	setup();
    while(1) loop();
}

void setup(void)
{

	boardInit();
	cli();
	redOff();
	greenOn();
	millis_init();
	gimbal.begin();
	servo_begin();
}

typedef enum accState_E
{
	eAccState_START,
	eAccState_INITIAL_RAMP,
	eAccState_HOLD,
	eAccState_NEXT_SPEED
}accState_T;

void loop(void)
{
	static int16_t currentSpeed = 0;
	static uint16_t lastServo = 0;
	uint16_t currentServo;
	static int16_t averageSpeed = 0;

	if (servo_changed())
	{
		currentServo = servo_value_uS();
		if ((currentServo-1500)>0)
		{
			asm("NOP");
		}
		if (currentServo != lastServo)
		{
			if (currentServo < 2000 && currentServo > 1000)
			{
				currentSpeed = (currentServo - 1500)*2;
				averageSpeed = ((averageSpeed*7) + (currentSpeed*3))/10;
				lastServo = currentServo;
				uint8_t powerScale = 1 + (3*abs(currentSpeed) /350);
				//uint8_t powerScale = 4;

				gimbal.set_PowerScale(powerScale);

			 	gimbal.set_speed_rpm(currentSpeed);
				//gimbal.set_speed_rpm(currentSpeed);
			}
		}
	}

	gimbal.tickle();
}

<commit_msg>tripolar - scaled speed up<commit_after>/*
 * tripolar.cpp
 *
 * Created: 5/12/2015 4:02:05 PM
 *  Author: Alan
 */

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& USER SETUP
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
	*/

	#define F_CPU 16000000UL
	//#define DO_DEBUG
/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& INCLUDES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

#include <avr/io.h>
#include "bldcGimbal.h"

#include "afro_nfet.h"
#include "fets.h"

#include <util/delay.h>
#include "millis.h"
#include <stdlib.h>

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& FUNCTION PROTOTYPES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

void setup(void);
void loop(void);

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& STRUCTURES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

/*****************************************************************************************************/
/* STRUCT: servoIsrData_S																			 */
/** Data structure used to interact with the Servo Interrupt service routine						 */
/*****************************************************************************************************/
typedef struct servoIsrData_S
{

	volatile uint16_t startTimeStamp;
	volatile uint16_t stopTimeStamp;
	bool dataReady;
	bool waitRising; //If true, we are waiting for a rising edge, otherwise a falling

}servoIsrData_T;

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& VARIABLES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

bldcGimbal gimbal;
static servoIsrData_T servoIsrData;

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& INTERRUPT SERVICE ROUTINES
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

ISR(TIMER1_CAPT_vect)
{
	sei();
	volatile uint16_t timeStamp;
	timeStamp = micros();
	if (servoIsrData.dataReady) return; //we already have a reading, so don't collect a new one till this one is read.

	if (servoIsrData.waitRising) {
		servoIsrData.startTimeStamp = timeStamp;  //If Rising Edge, record the start time
		TCCR1B &= ~_BV(ICES1);//Set interrupt for falling edge
		servoIsrData.waitRising = false;
	}
	else  //Otherwise its a falling edge so ..
	{
		servoIsrData.stopTimeStamp = timeStamp;	//Record the stop time..
		servoIsrData.dataReady = true;			//And signal that we have a value
		servoIsrData.waitRising = true;
		TCCR1B |= _BV(ICES1); //Set interrupt for rising edge
	}
}

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& SERVO ISR SUPPORT FUNCTIONS
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

inline void servo_begin(void)
{
	sei(); //Enable nested interrupts
	servoIsrData.dataReady = false;
	servoIsrData.waitRising = true;

	//Configure PCINT0 PIN (PB0) as an input
	DDRB &= ~_BV(DDB0);

	//DISABLE PULLUP RESISTOR (PB0)
	PORTB &= ~_BV(PORTB0);

	//Rising Edge Detect
	TCCR1B |= _BV(ICES1);

	//Enable Input Compare Interrupt
	TIMSK |= _BV(TICIE1);

}

inline bool servo_changed(void)
{
	if (servoIsrData.dataReady)
	{
		if (gimbal.motorPwm().icr1Conflict()) {
			TIMSK &= ~_BV(TICIE1);
			servoIsrData.dataReady = false;
			TIMSK |= _BV(TICIE1);
			redOn();
		}
	}
	return servoIsrData.dataReady;
}

inline uint16_t servo_value_uS(void)
{
	TIMSK &= ~_BV(TICIE1);
	uint16_t retVal = servoIsrData.stopTimeStamp - servoIsrData.startTimeStamp;
	TIMSK |= _BV(TICIE1);
	redOff();
	servoIsrData.dataReady = false;

	return retVal;
}

/*
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
&&& FUNCTIONS
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
*/

int main(void)
{
	setup();
    while(1) loop();
}

void setup(void)
{

	boardInit();
	cli();
	redOff();
	greenOn();
	millis_init();
	gimbal.begin();
	servo_begin();
}

typedef enum accState_E
{
	eAccState_START,
	eAccState_INITIAL_RAMP,
	eAccState_HOLD,
	eAccState_NEXT_SPEED
}accState_T;

void loop(void)
{
	static int16_t currentSpeed = 0;
	static int16_t lastServo = 0;
	int16_t currentServo;
	static int16_t averageSpeed = 0;

	if (servo_changed())
	{
		currentServo = servo_value_uS();
		if (abs(currentServo-1500)>1)
		{
			asm("NOP");
		}
		if (currentServo != lastServo)
		{
			if (currentServo < 2000 && currentServo > 1000)
			{
				currentSpeed = (11*(currentServo - 1500))/10;
				averageSpeed = ((averageSpeed*7) + (currentSpeed*3))/10;
				lastServo = currentServo;
				uint8_t powerScale = 1 + (3*abs(currentSpeed) /350);
				//uint8_t powerScale = 4;

				gimbal.set_PowerScale(powerScale);

			 	gimbal.set_speed_rpm(currentSpeed);
				//gimbal.set_speed_rpm(currentSpeed);
			}
		}
	}

	gimbal.tickle();
}

<|endoftext|>


<commit_before>#include <iostream>
#include <cassert>

#include "base64.h"

void runEncodeTests ()
{
  //test1, encode3({'a','a','a'}) == "YWFh"
  char testStr3[3];
  testStr3[0] = 'a';
  testStr3[1] = 'a';
  testStr3[2] = 'a';
  assert(base64Encode3Bytes(testStr3) == "YWFh");

  //test2, encode3({'x','y','z'}) == "eHl6"
  testStr3[0] = 'x';
  testStr3[1] = 'y';
  testStr3[2] = 'z';
  assert(base64Encode3Bytes(testStr3) == "eHl6");

  //test3, encode2({'a','a'}) == "YWE="
  char testStr2[2];
  testStr2[0] = 'a';
  testStr2[1] = 'a';
  assert(base64Encode2Bytes(testStr2) == "YWE=");
  assert(base64Encode(testStr2, 2) == "YWE=");

  //test4, encode2({'x','y'}) == "eHk=""
  testStr2[0] = 'x';
  testStr2[1] = 'y';
  assert(base64Encode2Bytes(testStr2) == "eHk=");
  assert(base64Encode(testStr2, 2) == "eHk=");

  //test5, encode1('a') == "YQ=="
  char testStr1[1];
  testStr1[0] = 'a';
  assert(base64Encode1Byte(testStr1[0]) == "YQ==");
  assert(base64Encode(testStr1, 1) == "YQ==");

  //test6, encode1('z') == "eg=="
  testStr1[0] = 'z';
  assert(base64Encode1Byte(testStr1[0]) == "eg==");
  assert(base64Encode(testStr1, 1) == "eg==");

  //test7, encodeStr({'a','b','c','d','e','f'}) == "YWJjZGVm"
  char testStr6[6];
  testStr6[0] = 'a';
  testStr6[1] = 'b';
  testStr6[2] = 'c';
  testStr6[3] = 'd';
  testStr6[4] = 'e';
  testStr6[5] = 'f';
  assert(base64Encode(testStr6, 6) == "YWJjZGVm");

  //test8, encodeStr({'u','v','w','x','y','z'}) == "dXZ3eHl6"
  testStr6[0] = 'u';
  testStr6[1] = 'v';
  testStr6[2] = 'w';
  testStr6[3] = 'x';
  testStr6[4] = 'y';
  testStr6[5] = 'z';
  assert(base64Encode(testStr6, 6) == "dXZ3eHl6");

  //test9, encodeStr({'a','b','c','d','e','f','g'}) == "YWJjZGVmZw=="
  char testStr7[7];
  testStr7[0] = 'a';
  testStr7[1] = 'b';
  testStr7[2] = 'c';
  testStr7[3] = 'd';
  testStr7[4] = 'e';
  testStr7[5] = 'f';
  testStr7[6] = 'g';
  assert(base64Encode(testStr7, 7) == "YWJjZGVmZw==");

  //test10, encodeStr({'t','u','v','w','x','y','z'}) == "dHV2d3h5eg=="
  testStr7[0] = 't';
  testStr7[1] = 'u';
  testStr7[2] = 'v';
  testStr7[3] = 'w';
  testStr7[4] = 'x';
  testStr7[5] = 'y';
  testStr7[6] = 'z';
  assert(base64Encode(testStr7, 7) == "dHV2d3h5eg==");

  //test11, encodeStr({'a','b','c','d','e','f','g','h'}) == "YWJjZGVmZ2g="
  char testStr8[8];
  testStr8[0] = 'a';
  testStr8[1] = 'b';
  testStr8[2] = 'c';
  testStr8[3] = 'd';
  testStr8[4] = 'e';
  testStr8[5] = 'f';
  testStr8[6] = 'g';
  testStr8[7] = 'h';
  assert(base64Encode(testStr8, 8) == "YWJjZGVmZ2g=");

  //test12, encodeStr({'s','t','u','v','w','x','y','z'}) == "c3R1dnd4eXo="
  testStr8[0] = 's';
  testStr8[1] = 't';
  testStr8[2] = 'u';
  testStr8[3] = 'v';
  testStr8[4] = 'w';
  testStr8[5] = 'x';
  testStr8[6] = 'y';
  testStr8[7] = 'z';
  assert(base64Encode(testStr8, 8) == "c3R1dnd4eXo=");

  std::cout << "Encode tests pass!" << std::endl;
}

void runDecodeTests ()
{
  //test1, test decode table
  std::string base64CharsNoEquals = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                                    "abcdefghijklmnopqrstuvwxyz"
                                    "0123456789+/";
  for ( unsigned int i = 0; i < base64CharsNoEquals.length(); i++ )
  {
    unsigned int base64Code =
        getDecodedBase64Char(base64CharsNoEquals[i]);
    assert(i == base64Code);
  }

  //test2, decode4({'Y','W','F','h'}) == "aaa"
  char testStr4[4];
  testStr4[0] = 'Y';
  testStr4[1] = 'W';
  testStr4[2] = 'F';
  testStr4[3] = 'h';
  assert(base64Decode4Bytes(testStr4) == "aaa");
  assert(base64Decode("YWFh") == "aaa");

  //test3, decode4({'Y','W','F','='}) == "aa"
  testStr4[0] = 'Y';
  testStr4[1] = 'W';
  testStr4[2] = 'F';
  testStr4[3] = '=';
  assert(base64Decode4Bytes(testStr4) == "aa");
  assert(base64Decode("YWF=") == "aa");

  //test4, decode4({'Y','W','=','='}) == "a"
  testStr4[0] = 'Y';
  testStr4[1] = 'W';
  testStr4[2] = '=';
  testStr4[3] = '=';
  assert(base64Decode4Bytes(testStr4) == "a");
  assert(base64Decode("YW==") == "a");

  //test5, decode({'e','H','l','6'}) == "xyz"
  testStr4[0] = 'e';
  testStr4[1] = 'H';
  testStr4[2] = 'l';
  testStr4[3] = '6';
  assert(base64Decode4Bytes(testStr4) == "xyz");
  assert(base64Decode("eHl6") == "xyz");

  //test6, decode({'e','H','l','='}) == "xy"
  testStr4[0] = 'e';
  testStr4[1] = 'H';
  testStr4[2] = 'l';
  testStr4[3] = '=';
  assert(base64Decode4Bytes(testStr4) == "xy");
  assert(base64Decode("eHl=") == "xy");

  //test7, decode({'e','H','=','='}) == "x"
  testStr4[0] = 'e';
  testStr4[1] = 'H';
  testStr4[2] = '=';
  testStr4[3] = '=';
  assert(base64Decode4Bytes(testStr4) == "x");
  assert(base64Decode("eH==") == "x");

  //test8, decode("YWJjZGVm") == "abcdef"
  assert(base64Decode("YWJjZGVm") == "abcdef");

  //test9, decode("dXZ3eHl6") = "uvwxyz"
  assert(base64Decode("dXZ3eHl6") == "uvwxyz");

  //test10, decode("dnd4eXo=") == "vwxyz"
  assert(base64Decode("dnd4eXo=") == "vwxyz");

  //test11, decode("d3h5eg==") = "wxyz"
  assert(base64Decode("d3h5eg==") == "wxyz");

  std::cout << "Decode tests pass!" << std::endl;
}

void runCombinedTests()
{
  std::string resultStr;

  //test1, encode and decode the alphabet in lover case
  std::string alphabet0 = "abcdefghijklmnopqrstuvwxyz";
  resultStr =
      base64Decode(
          base64Encode( alphabet0.c_str(), alphabet0.length() )
      );
  assert(resultStr  == alphabet0);

  //test2, encode and decode more characters
  std::string alphabet1 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                          "abcdefghijklmnopqrstuvwxyz"
                          "0123456789"
                          "!@#$^&*()-_=+[]{}:;<>,.?/|~`";
  resultStr =
      base64Decode(
          base64Encode(alphabet1.c_str(), alphabet1.length() )
      );

  assert(resultStr == alphabet1);

  std::cout << "Combined tests pass!" << std::endl;
}

int main ( )
{
  runEncodeTests();
  runDecodeTests();
  runCombinedTests();

  return 0;
}<commit_msg>eof \n<commit_after>#include <iostream>
#include <cassert>

#include "base64.h"

void runEncodeTests ()
{
  //test1, encode3({'a','a','a'}) == "YWFh"
  char testStr3[3];
  testStr3[0] = 'a';
  testStr3[1] = 'a';
  testStr3[2] = 'a';
  assert(base64Encode3Bytes(testStr3) == "YWFh");

  //test2, encode3({'x','y','z'}) == "eHl6"
  testStr3[0] = 'x';
  testStr3[1] = 'y';
  testStr3[2] = 'z';
  assert(base64Encode3Bytes(testStr3) == "eHl6");

  //test3, encode2({'a','a'}) == "YWE="
  char testStr2[2];
  testStr2[0] = 'a';
  testStr2[1] = 'a';
  assert(base64Encode2Bytes(testStr2) == "YWE=");
  assert(base64Encode(testStr2, 2) == "YWE=");

  //test4, encode2({'x','y'}) == "eHk=""
  testStr2[0] = 'x';
  testStr2[1] = 'y';
  assert(base64Encode2Bytes(testStr2) == "eHk=");
  assert(base64Encode(testStr2, 2) == "eHk=");

  //test5, encode1('a') == "YQ=="
  char testStr1[1];
  testStr1[0] = 'a';
  assert(base64Encode1Byte(testStr1[0]) == "YQ==");
  assert(base64Encode(testStr1, 1) == "YQ==");

  //test6, encode1('z') == "eg=="
  testStr1[0] = 'z';
  assert(base64Encode1Byte(testStr1[0]) == "eg==");
  assert(base64Encode(testStr1, 1) == "eg==");

  //test7, encodeStr({'a','b','c','d','e','f'}) == "YWJjZGVm"
  char testStr6[6];
  testStr6[0] = 'a';
  testStr6[1] = 'b';
  testStr6[2] = 'c';
  testStr6[3] = 'd';
  testStr6[4] = 'e';
  testStr6[5] = 'f';
  assert(base64Encode(testStr6, 6) == "YWJjZGVm");

  //test8, encodeStr({'u','v','w','x','y','z'}) == "dXZ3eHl6"
  testStr6[0] = 'u';
  testStr6[1] = 'v';
  testStr6[2] = 'w';
  testStr6[3] = 'x';
  testStr6[4] = 'y';
  testStr6[5] = 'z';
  assert(base64Encode(testStr6, 6) == "dXZ3eHl6");

  //test9, encodeStr({'a','b','c','d','e','f','g'}) == "YWJjZGVmZw=="
  char testStr7[7];
  testStr7[0] = 'a';
  testStr7[1] = 'b';
  testStr7[2] = 'c';
  testStr7[3] = 'd';
  testStr7[4] = 'e';
  testStr7[5] = 'f';
  testStr7[6] = 'g';
  assert(base64Encode(testStr7, 7) == "YWJjZGVmZw==");

  //test10, encodeStr({'t','u','v','w','x','y','z'}) == "dHV2d3h5eg=="
  testStr7[0] = 't';
  testStr7[1] = 'u';
  testStr7[2] = 'v';
  testStr7[3] = 'w';
  testStr7[4] = 'x';
  testStr7[5] = 'y';
  testStr7[6] = 'z';
  assert(base64Encode(testStr7, 7) == "dHV2d3h5eg==");

  //test11, encodeStr({'a','b','c','d','e','f','g','h'}) == "YWJjZGVmZ2g="
  char testStr8[8];
  testStr8[0] = 'a';
  testStr8[1] = 'b';
  testStr8[2] = 'c';
  testStr8[3] = 'd';
  testStr8[4] = 'e';
  testStr8[5] = 'f';
  testStr8[6] = 'g';
  testStr8[7] = 'h';
  assert(base64Encode(testStr8, 8) == "YWJjZGVmZ2g=");

  //test12, encodeStr({'s','t','u','v','w','x','y','z'}) == "c3R1dnd4eXo="
  testStr8[0] = 's';
  testStr8[1] = 't';
  testStr8[2] = 'u';
  testStr8[3] = 'v';
  testStr8[4] = 'w';
  testStr8[5] = 'x';
  testStr8[6] = 'y';
  testStr8[7] = 'z';
  assert(base64Encode(testStr8, 8) == "c3R1dnd4eXo=");

  std::cout << "Encode tests pass!" << std::endl;
}

void runDecodeTests ()
{
  //test1, test decode table
  std::string base64CharsNoEquals = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                                    "abcdefghijklmnopqrstuvwxyz"
                                    "0123456789+/";
  for ( unsigned int i = 0; i < base64CharsNoEquals.length(); i++ )
  {
    unsigned int base64Code =
        getDecodedBase64Char(base64CharsNoEquals[i]);
    assert(i == base64Code);
  }

  //test2, decode4({'Y','W','F','h'}) == "aaa"
  char testStr4[4];
  testStr4[0] = 'Y';
  testStr4[1] = 'W';
  testStr4[2] = 'F';
  testStr4[3] = 'h';
  assert(base64Decode4Bytes(testStr4) == "aaa");
  assert(base64Decode("YWFh") == "aaa");

  //test3, decode4({'Y','W','F','='}) == "aa"
  testStr4[0] = 'Y';
  testStr4[1] = 'W';
  testStr4[2] = 'F';
  testStr4[3] = '=';
  assert(base64Decode4Bytes(testStr4) == "aa");
  assert(base64Decode("YWF=") == "aa");

  //test4, decode4({'Y','W','=','='}) == "a"
  testStr4[0] = 'Y';
  testStr4[1] = 'W';
  testStr4[2] = '=';
  testStr4[3] = '=';
  assert(base64Decode4Bytes(testStr4) == "a");
  assert(base64Decode("YW==") == "a");

  //test5, decode({'e','H','l','6'}) == "xyz"
  testStr4[0] = 'e';
  testStr4[1] = 'H';
  testStr4[2] = 'l';
  testStr4[3] = '6';
  assert(base64Decode4Bytes(testStr4) == "xyz");
  assert(base64Decode("eHl6") == "xyz");

  //test6, decode({'e','H','l','='}) == "xy"
  testStr4[0] = 'e';
  testStr4[1] = 'H';
  testStr4[2] = 'l';
  testStr4[3] = '=';
  assert(base64Decode4Bytes(testStr4) == "xy");
  assert(base64Decode("eHl=") == "xy");

  //test7, decode({'e','H','=','='}) == "x"
  testStr4[0] = 'e';
  testStr4[1] = 'H';
  testStr4[2] = '=';
  testStr4[3] = '=';
  assert(base64Decode4Bytes(testStr4) == "x");
  assert(base64Decode("eH==") == "x");

  //test8, decode("YWJjZGVm") == "abcdef"
  assert(base64Decode("YWJjZGVm") == "abcdef");

  //test9, decode("dXZ3eHl6") = "uvwxyz"
  assert(base64Decode("dXZ3eHl6") == "uvwxyz");

  //test10, decode("dnd4eXo=") == "vwxyz"
  assert(base64Decode("dnd4eXo=") == "vwxyz");

  //test11, decode("d3h5eg==") = "wxyz"
  assert(base64Decode("d3h5eg==") == "wxyz");

  std::cout << "Decode tests pass!" << std::endl;
}

void runCombinedTests()
{
  std::string resultStr;

  //test1, encode and decode the alphabet in lover case
  std::string alphabet0 = "abcdefghijklmnopqrstuvwxyz";
  resultStr =
      base64Decode(
          base64Encode( alphabet0.c_str(), alphabet0.length() )
      );
  assert(resultStr  == alphabet0);

  //test2, encode and decode more characters
  std::string alphabet1 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                          "abcdefghijklmnopqrstuvwxyz"
                          "0123456789"
                          "!@#$^&*()-_=+[]{}:;<>,.?/|~`";
  resultStr =
      base64Decode(
          base64Encode(alphabet1.c_str(), alphabet1.length() )
      );

  assert(resultStr == alphabet1);

  std::cout << "Combined tests pass!" << std::endl;
}

int main ( )
{
  runEncodeTests();
  runDecodeTests();
  runCombinedTests();

  return 0;
}
<|endoftext|>


<commit_before>//-----------------------------------------------------------------------------
//
// Copyright (C) 2015 David Hill
//
// See COPYING for license information.
//
//-----------------------------------------------------------------------------
//
// PrintBuf class.
//
//-----------------------------------------------------------------------------

#include "PrintBuf.hpp"

#include "BinaryIO.hpp"

#include <cstdio>
#include <cstdlib>
#include <new>

//----------------------------------------------------------------------------|
// Extern Functions                                                           |
//

namespace ACSVM
{
   //
   // PrintBuf constructor
   //
   PrintBuf::PrintBuf() :
      buffer{nullptr},
      bufEnd{nullptr},
      bufBeg{nullptr},
      bufPtr{nullptr}
   {
   }

   //
   // PrintBuf destructor
   //
   PrintBuf::~PrintBuf()
   {
      std::free(buffer);
   }

   //
   // PrintBuf::drop
   //
   void PrintBuf::drop()
   {
      if(bufBeg != buffer)
      {
         bufPtr = bufBeg - 4;
         bufBeg = bufPtr - ReadLE4(reinterpret_cast<Byte *>(bufPtr));
      }
      else
         bufPtr = bufBeg;
   }

   //
   // PrintBuf::format
   //
   void PrintBuf::format(char const *fmt, ...)
   {
      va_list arg;
      va_start(arg, fmt);
      formatv(fmt, arg);
      va_end(arg);
   }

   //
   // PrintBuf::formatv
   //
   void PrintBuf::formatv(char const *fmt, va_list arg)
   {
      bufPtr += std::vsprintf(bufPtr, fmt, arg);
   }

   //
   // PrintBuf::push
   //
   void PrintBuf::push()
   {
      reserve(4);
      WriteLE4(reinterpret_cast<Byte *>(bufPtr), bufPtr - bufBeg);
      bufBeg = bufPtr += 4;
   }

   //
   // PrintBuf::reserve
   //
   void PrintBuf::reserve(std::size_t count)
   {
      if(static_cast<std::size_t>(bufEnd - bufPtr) > count)
         return;

      // Allocate extra to anticipate further reserves. +1 for null.
      count = count * 2 + 1;

      std::size_t idxEnd = bufEnd - buffer;
      std::size_t idxBeg = bufBeg - buffer;
      std::size_t idxPtr = bufPtr - buffer;

      // Check for size overflow.
      if(SIZE_MAX - idxEnd < count)
         throw std::bad_alloc();

      // Check that the current segment won't pass the push limit.
      if(UINT32_MAX - (idxEnd - idxBeg) < count)
         throw std::bad_alloc();

      idxEnd += count;

      if(!(buffer = static_cast<char *>(std::realloc(buffer, idxEnd))))
         throw std::bad_alloc();

      bufEnd = buffer + idxEnd;
      bufBeg = buffer + idxBeg;
      bufPtr = buffer + idxPtr;
   }
}

// EOF

<commit_msg>Fixed memory leak on realloc failure in PrintBuf::reserve.<commit_after>//-----------------------------------------------------------------------------
//
// Copyright (C) 2015 David Hill
//
// See COPYING for license information.
//
//-----------------------------------------------------------------------------
//
// PrintBuf class.
//
//-----------------------------------------------------------------------------

#include "PrintBuf.hpp"

#include "BinaryIO.hpp"

#include <cstdio>
#include <cstdlib>
#include <new>

//----------------------------------------------------------------------------|
// Extern Functions                                                           |
//

namespace ACSVM
{
   //
   // PrintBuf constructor
   //
   PrintBuf::PrintBuf() :
      buffer{nullptr},
      bufEnd{nullptr},
      bufBeg{nullptr},
      bufPtr{nullptr}
   {
   }

   //
   // PrintBuf destructor
   //
   PrintBuf::~PrintBuf()
   {
      std::free(buffer);
   }

   //
   // PrintBuf::drop
   //
   void PrintBuf::drop()
   {
      if(bufBeg != buffer)
      {
         bufPtr = bufBeg - 4;
         bufBeg = bufPtr - ReadLE4(reinterpret_cast<Byte *>(bufPtr));
      }
      else
         bufPtr = bufBeg;
   }

   //
   // PrintBuf::format
   //
   void PrintBuf::format(char const *fmt, ...)
   {
      va_list arg;
      va_start(arg, fmt);
      formatv(fmt, arg);
      va_end(arg);
   }

   //
   // PrintBuf::formatv
   //
   void PrintBuf::formatv(char const *fmt, va_list arg)
   {
      bufPtr += std::vsprintf(bufPtr, fmt, arg);
   }

   //
   // PrintBuf::push
   //
   void PrintBuf::push()
   {
      reserve(4);
      WriteLE4(reinterpret_cast<Byte *>(bufPtr), bufPtr - bufBeg);
      bufBeg = bufPtr += 4;
   }

   //
   // PrintBuf::reserve
   //
   void PrintBuf::reserve(std::size_t count)
   {
      if(static_cast<std::size_t>(bufEnd - bufPtr) > count)
         return;

      // Allocate extra to anticipate further reserves. +1 for null.
      count = count * 2 + 1;

      std::size_t idxEnd = bufEnd - buffer;
      std::size_t idxBeg = bufBeg - buffer;
      std::size_t idxPtr = bufPtr - buffer;

      // Check for size overflow.
      if(SIZE_MAX - idxEnd < count)
         throw std::bad_alloc();

      // Check that the current segment won't pass the push limit.
      if(UINT32_MAX - (idxEnd - idxBeg) < count)
         throw std::bad_alloc();

      idxEnd += count;

      char *bufNew;
      if(!(bufNew = static_cast<char *>(std::realloc(buffer, idxEnd))))
         throw std::bad_alloc();

      buffer = bufNew;
      bufEnd = buffer + idxEnd;
      bufBeg = buffer + idxBeg;
      bufPtr = buffer + idxPtr;
   }
}

// EOF

<|endoftext|>


<commit_before>/*
 * Copyright (c) 2011 James Peach
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <ts/ts.h>
#include <spdy/spdy.h>
#include <platform/logging.h>
#include "io.h"

static int spdy_session_io(TSCont, TSEvent, void *);
static TSMLoc make_ts_http_request(TSMBuffer, const spdy::key_value_block&);
static void print_ts_http_header(TSMBuffer, TSMLoc);

typedef scoped_ts_object<TSMBuffer, TSMBufferCreate, TSMBufferDestroy> scoped_mbuffer;

struct scoped_http_header
{
    explicit scoped_http_header(TSMBuffer b)
            : header(TS_NULL_MLOC), buffer(b) {
        header = TSHttpHdrCreate(buffer);
    }

    ~scoped_http_header() {
        TSHttpHdrDestroy(buffer, header);
        TSHandleMLocRelease(buffer, TS_NULL_MLOC, header);
    }

    operator bool() const { return header != TS_NULL_MLOC; }
    TSMLoc get() { return header; }

private:
    TSMLoc      header;
    TSMBuffer   buffer;
};

static void
resolve_host_name(spdy_io_stream * stream, const std::string& hostname)
{
    TSCont contp;

    contp = TSContCreate(spdy_session_io, TSMutexCreate());
    TSContDataSet(contp, stream);

    // XXX split the host and port and stash the port in the resulting sockaddr
    stream->action = TSHostLookup(contp, hostname.c_str(), hostname.size());
}

static bool
initiate_client_request(
        spdy_io_stream * stream)
{
    scoped_mbuffer  buffer;
    TSMLoc          header;

    header = make_ts_http_request(buffer.get(), stream->kvblock);
    if (header == TS_NULL_MLOC) {
        return false;
    }

    print_ts_http_header(buffer.get(), header);

    // Need the kv:version and kv:method to actually make the request. It looks
    // like the most straightforward way is to build the HTTP request by hand
    // and pump it into TSFetchUrl().

    // Apparantly TSFetchUrl may have performance problems,
    // see https://issues.apache.org/jira/browse/TS-912

    // We probably need to manually do the caching, using
    // TSCacheRead/TSCacheWrite.

    // For POST requests which may contain a lot of data, we probably need to
    // do a bunch of work. Looks like the recommended path is
    //      TSHttpConnect()
    //      TSVConnWrite() to send an HTTP request.
    //      TSVConnRead() to get the HTTP response.
    //      TSHttpParser to parse the response (if needed).

    return true;
}

static int
spdy_session_io(TSCont contp, TSEvent ev, void * edata)
{
    TSHostLookupResult dns = (TSHostLookupResult)edata;
    spdy_io_stream * stream = spdy_io_stream::get(contp);

    switch (ev) {
    case TS_EVENT_HOST_LOOKUP:
        if (dns) {
            const struct sockaddr * addr;
            addr = TSHostLookupResultAddrGet(dns);
            debug_http("resolved %s => %s",
                    stream->kvblock.url().hostport.c_str(), cstringof(*addr));
            initiate_client_request(stream);
        } else {
            // XXX
            // Experimentally, if the DNS lookup fails, web proxies return 502
            // Bad Gateway. We should cobble up a HTTP response to tunnel back
            // in a SYN_REPLY.
        }

        stream->action = NULL;
        break;

    default:
        debug_plugin("unexpected accept event %s", cstringof(ev));
    }

    return TS_EVENT_NONE;
}

static void
print_ts_http_header(TSMBuffer buffer, TSMLoc header)
{
    spdy_io_control::buffered_stream iobuf;
    int64_t nbytes;
    int64_t avail;
    const char * ptr;
    TSIOBufferBlock blk;

    TSHttpHdrPrint(buffer, header, iobuf.buffer);
    blk = TSIOBufferStart(iobuf.buffer);
    avail = TSIOBufferBlockReadAvail(blk, iobuf.reader);
    ptr = (const char *)TSIOBufferBlockReadStart(blk, iobuf.reader, &nbytes);

    debug_http("http request (%zu of %zu bytes):\n%*.*s",
            nbytes, avail, (int)nbytes, (int)nbytes, ptr);
}

static void
make_ts_http_url(
        TSMBuffer   buffer,
        TSMLoc      header,
        const spdy::key_value_block& kvblock)
{
    TSReturnCode    tstatus;
    TSMLoc          url;

    tstatus = TSHttpHdrUrlGet(buffer, header, &url);
    if (tstatus == TS_ERROR) {
        tstatus = TSUrlCreate(buffer, &url);
    }

    TSUrlSchemeSet(buffer, url,
            kvblock.url().scheme.data(), kvblock.url().scheme.size());
    TSUrlHostSet(buffer, url,
            kvblock.url().hostport.data(), kvblock.url().hostport.size());
    TSUrlPathSet(buffer, url,
            kvblock.url().path.data(), kvblock.url().path.size());
    TSHttpHdrMethodSet(buffer, header,
            kvblock.url().method.data(), kvblock.url().method.size());

    TSHttpHdrUrlSet(buffer, header, url);

    TSAssert(tstatus == TS_SUCCESS);
}

static TSMLoc
make_ts_http_request(
        TSMBuffer buffer,
        const spdy::key_value_block& kvblock)
{

    TSMLoc header = TSHttpHdrCreate(buffer);

    TSHttpHdrTypeSet(buffer, header, TS_HTTP_TYPE_REQUEST);

    // XXX extract the real HTTP version header from kvblock.url()
    TSHttpHdrVersionSet(buffer, header, TS_HTTP_VERSION(1, 1));
    make_ts_http_url(buffer, header, kvblock);

    // Duplicate the header fields into the MIME header for the HTTP request we
    // are building.
    for (auto ptr(kvblock.begin()); ptr != kvblock.end(); ++ptr) {
        if (ptr->first[0] != ':') {
            TSMLoc field;

            // XXX Need special handling for duplicate headers; we should
            // append them as a multi-value

            TSMimeHdrFieldCreateNamed(buffer, header,
                    ptr->first.c_str(), -1, &field);
            TSMimeHdrFieldValueStringInsert(buffer, header, field,
                    -1, ptr->second.c_str(), -1);
            TSMimeHdrFieldAppend(buffer, header, field);
        }
    }

    return header;
}

spdy_io_stream::spdy_io_stream(unsigned s)
    : stream_id(s), action(NULL), kvblock()
{
}

spdy_io_stream::~spdy_io_stream()
{
    if (action) {
        TSActionCancel(action);
    }
}

void
spdy_io_stream::start()
{
    resolve_host_name(this, kvblock.url().hostport);
}

/* vim: set sw=4 ts=4 tw=79 et : */
<commit_msg>Send the SPDY request through the ATS client machinery.<commit_after>/*
 * Copyright (c) 2011 James Peach
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <ts/ts.h>
#include <spdy/spdy.h>
#include <platform/logging.h>
#include <netinet/in.h>
#include "io.h"

#define SPDY_EVENT_HTTP_SUCCESS     90000
#define SPDY_EVENT_HTTP_FAILURE     90001
#define SPDY_EVENT_HTTP_TIMEOUT     90002

static int spdy_session_io(TSCont, TSEvent, void *);
static TSMLoc make_ts_http_request(TSMBuffer, const spdy::key_value_block&);
static void print_ts_http_header(TSMBuffer, TSMLoc);

typedef scoped_ts_object<TSMBuffer, TSMBufferCreate, TSMBufferDestroy> scoped_mbuffer;

struct scoped_http_header
{
    explicit scoped_http_header(TSMBuffer b)
            : header(TS_NULL_MLOC), buffer(b) {
        header = TSHttpHdrCreate(buffer);
    }

    scoped_http_header(TSMBuffer b, TSMLoc h)
            : header(h), buffer(b) {
    }

    ~scoped_http_header() {
        if (header != TS_NULL_MLOC) {
            TSHttpHdrDestroy(buffer, header);
            TSHandleMLocRelease(buffer, TS_NULL_MLOC, header);
        }
    }

    operator bool() const {
        return header != TS_NULL_MLOC;
    }

    operator TSMLoc() const {
        return header;
    }

    TSMLoc get() {
        return header;
    }

    TSMLoc release() {
        TSMLoc tmp = TS_NULL_MLOC;
        std::swap(tmp, header);
        return tmp;
    }

private:
    TSMLoc      header;
    TSMBuffer   buffer;
};

struct inet_address
{
    explicit inet_address(const struct sockaddr * addr) {
        memcpy(&sa.storage, addr, addr->sa_len);
    }

    uint16_t& port() {
        switch (sa.storage.ss_family) {
        case AF_INET:
            return sa.in.sin_port;
        case AF_INET6:
            return sa.in6.sin6_port;
        default:
            TSReleaseAssert("invalid inet address type");
            return sa.in.sin_port;
        }
    }

    const sockaddr * saddr() const {
        return &sa.sa;
    }

private:
    union {
        struct sockaddr_in  in;
        struct sockaddr_in6 in6;
        struct sockaddr     sa;
        struct sockaddr_storage storage;
    } sa;
};

template <> std::string
stringof<inet_address>(const inet_address& inaddr) {
    return cstringof(*inaddr.saddr());
}

static void
resolve_host_name(spdy_io_stream * stream, const std::string& hostname)
{
    TSCont contp;

    contp = TSContCreate(spdy_session_io, TSMutexCreate());
    TSContDataSet(contp, stream);

    // XXX split the host and port and stash the port in the resulting sockaddr
    stream->action = TSHostLookup(contp, hostname.c_str(), hostname.size());
}

static bool
initiate_client_request(
        spdy_io_stream *        stream,
        const struct sockaddr * addr,
        TSCont                  contp)

{
    scoped_mbuffer      buffer;
    spdy_io_control::buffered_stream iobuf;
    int64_t nbytes;
    const char * ptr;
    TSIOBufferBlock blk;

    scoped_http_header header(
            buffer.get(), make_ts_http_request(buffer.get(), stream->kvblock));
    if (!header) {
        return false;
    }

    print_ts_http_header(buffer.get(), header);

    // Need the kv:version and kv:method to actually make the request. It looks
    // like the most straightforward way is to build the HTTP request by hand
    // and pump it into TSFetchUrl().

    // Apparantly TSFetchUrl may have performance problems,
    // see https://issues.apache.org/jira/browse/TS-912

    // We probably need to manually do the caching, using
    // TSCacheRead/TSCacheWrite.

    // For POST requests which may contain a lot of data, we probably need to
    // do a bunch of work. Looks like the recommended path is
    //      TSHttpConnect()
    //      TSVConnWrite() to send an HTTP request.
    //      TSVConnRead() to get the HTTP response.
    //      TSHttpParser to parse the response (if needed).

    TSFetchEvent events = {
        /* success_event_id */ SPDY_EVENT_HTTP_SUCCESS,
        /* failure_event_id */ SPDY_EVENT_HTTP_FAILURE,
        /* timeout_event_id */ SPDY_EVENT_HTTP_TIMEOUT
    };

    TSHttpHdrPrint(buffer.get(), header, iobuf.buffer);
    blk = TSIOBufferStart(iobuf.buffer);
    ptr = (const char *)TSIOBufferBlockReadStart(blk, iobuf.reader, &nbytes);

    TSFetchUrl(ptr, nbytes, addr, contp, AFTER_BODY, events);
    return true;
}

static int
spdy_session_io(TSCont contp, TSEvent ev, void * edata)
{
    TSHostLookupResult dns = (TSHostLookupResult)edata;
    spdy_io_stream * stream = spdy_io_stream::get(contp);

    switch (ev) {
    case TS_EVENT_HOST_LOOKUP:
        if (dns) {
            inet_address addr(TSHostLookupResultAddrGet(dns));
            debug_http("resolved %s => %s",
                    stream->kvblock.url().hostport.c_str(), cstringof(addr));
            addr.port() = htons(80); // XXX should be parsed from hostport
            initiate_client_request(stream, addr.saddr(), contp);
        } else {
            // XXX
            // Experimentally, if the DNS lookup fails, web proxies return 502
            // Bad Gateway. We should cobble up a HTTP response to tunnel back
            // in a SYN_REPLY.
        }

        stream->action = NULL;
        break;

    case SPDY_EVENT_HTTP_SUCCESS:
        debug_http("HTTP success event");
        break;

    case SPDY_EVENT_HTTP_FAILURE:
        debug_http("HTTP failure event");
        break;

    case SPDY_EVENT_HTTP_TIMEOUT:
        debug_http("HTTP timeout event");
        break;

    default:
        debug_plugin("unexpected accept event %s", cstringof(ev));
    }

    return TS_EVENT_NONE;
}

static void
print_ts_http_header(TSMBuffer buffer, TSMLoc header)
{
    spdy_io_control::buffered_stream iobuf;
    int64_t nbytes;
    int64_t avail;
    const char * ptr;
    TSIOBufferBlock blk;

    TSHttpHdrPrint(buffer, header, iobuf.buffer);
    blk = TSIOBufferStart(iobuf.buffer);
    avail = TSIOBufferBlockReadAvail(blk, iobuf.reader);
    ptr = (const char *)TSIOBufferBlockReadStart(blk, iobuf.reader, &nbytes);

    debug_http("http request (%zu of %zu bytes):\n%*.*s",
            nbytes, avail, (int)nbytes, (int)nbytes, ptr);
}

static void
make_ts_http_url(
        TSMBuffer   buffer,
        TSMLoc      header,
        const spdy::key_value_block& kvblock)
{
    TSReturnCode    tstatus;
    TSMLoc          url;

    tstatus = TSHttpHdrUrlGet(buffer, header, &url);
    if (tstatus == TS_ERROR) {
        tstatus = TSUrlCreate(buffer, &url);
    }

    TSUrlSchemeSet(buffer, url,
            kvblock.url().scheme.data(), kvblock.url().scheme.size());
    TSUrlHostSet(buffer, url,
            kvblock.url().hostport.data(), kvblock.url().hostport.size());
    TSUrlPathSet(buffer, url,
            kvblock.url().path.data(), kvblock.url().path.size());
    TSHttpHdrMethodSet(buffer, header,
            kvblock.url().method.data(), kvblock.url().method.size());

    TSHttpHdrUrlSet(buffer, header, url);

    TSAssert(tstatus == TS_SUCCESS);
}

static TSMLoc
make_ts_http_request(
        TSMBuffer buffer,
        const spdy::key_value_block& kvblock)
{

    scoped_http_header header(buffer);

    TSHttpHdrTypeSet(buffer, header, TS_HTTP_TYPE_REQUEST);

    // XXX extract the real HTTP version header from kvblock.url()
    TSHttpHdrVersionSet(buffer, header, TS_HTTP_VERSION(1, 1));
    make_ts_http_url(buffer, header, kvblock);

    // Duplicate the header fields into the MIME header for the HTTP request we
    // are building.
    for (auto ptr(kvblock.begin()); ptr != kvblock.end(); ++ptr) {
        if (ptr->first[0] != ':') {
            TSMLoc field;

            // XXX Need special handling for duplicate headers; we should
            // append them as a multi-value

            TSMimeHdrFieldCreateNamed(buffer, header,
                    ptr->first.c_str(), -1, &field);
            TSMimeHdrFieldValueStringInsert(buffer, header, field,
                    -1, ptr->second.c_str(), -1);
            TSMimeHdrFieldAppend(buffer, header, field);
        }
    }

    return header.release();
}

spdy_io_stream::spdy_io_stream(unsigned s)
    : stream_id(s), action(NULL), kvblock()
{
}

spdy_io_stream::~spdy_io_stream()
{
    if (action) {
        TSActionCancel(action);
    }
}

void
spdy_io_stream::start()
{
    resolve_host_name(this, kvblock.url().hostport);
}

/* vim: set sw=4 ts=4 tw=79 et : */
<|endoftext|>