<commit_before>/**
 * Copyright (c) 2016-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

#include "DedupBlocksPass.h"

#include <boost/optional.hpp>
#include <iterator>
#include <unordered_map>
#include <unordered_set>

#include "ControlFlow.h"
#include "DexClass.h"
#include "DexOutput.h"
#include "DexUtil.h"
#include "IRCode.h"
#include "Resolver.h"
#include "Transform.h"
#include "Walkers.h"

/*
 * This pass removes blocks that are duplicates in a method.
 *
 * If a method has multiple blocks with the same code and the same successors,
 * delete all but one of the blocks. Naming one of them the canonical block.
 *
 * Then, reroute all the predecessors of all the blocks to that canonical block.
 */

namespace {

using hash_t = std::size_t;

struct BlockAsKey {
  IRCode* code;
  Block* block;

  BlockAsKey(IRCode* c, Block* b) : code(c), block(b) {}

  bool operator==(const BlockAsKey& other) const {
    return same_successors(other) && same_try_regions(other) &&
           same_code(other);
  }

  // Structural equality of opcodes except branch targets are ignored
  // because they are unknown until we sync back to DexInstructions.
  bool same_code(const BlockAsKey& other) const {
    return InstructionIterable(block).structural_equals(
        InstructionIterable(other.block));
  }

  // The blocks must also have the exact same successors
  // (and reached in the same ways)
  bool same_successors(const BlockAsKey& other) const {
    const auto& b1_succs = this->block->succs();
    const auto& b2_succs = other.block->succs();
    if (b1_succs.size() != b2_succs.size()) {
      return false;
    }
    for (Block* b1_succ : b1_succs) {
      const auto& in_b2 = std::find(b2_succs.begin(), b2_succs.end(), b1_succ);
      if (in_b2 == b2_succs.end()) {
        // b1 has a succ that b2 doesn't
        return false;
      }

      if (code->cfg().edge(this->block, b1_succ) !=
          code->cfg().edge(other.block, *in_b2)) {
        // successors are the same but the type of connecting
        // edges are different.
        // Like so:
        //
        // B0: if-eqz v0, B3
        // B1: return
        // B2: if eqz v0, B1
        // B3: add-int v1, 1
        //
        // B0's succs are B1 (fallthru) and B3 (branch)
        // B2's succs are B3 (fallthru) and B1 (branch)
        return false;
      }
    }
    return true;
  }

  // return true if in the same try region (or both in no try region at all)
  bool same_try_regions(const BlockAsKey& other) const {
    return transform::find_active_catch(code, this->block->begin()) ==
           transform::find_active_catch(code, other.block->begin());
  }
};

struct BlockHasher {
  hash_t operator()(const BlockAsKey& key) const {
    hash_t result = 0;
    for (auto& mie : InstructionIterable(key.block)) {
      result ^= mie.insn->hash();
    }
    return result;
  }
};

class DedupBlocksImpl {
 public:
  DedupBlocksImpl(const std::vector<DexClass*>& scope,
                  PassManager& mgr,
                  const DedupBlocksPass::Config& config)
      : m_scope(scope), m_mgr(mgr), m_config(config) {}

  void run() {
    walk_methods(m_scope, [this](DexMethod* method) {
      if (m_config.method_black_list.count(method) != 0) {
        return;
      }

      IRCode* code = method->get_code();
      if (code == nullptr) {
        return;
      }
      code->build_cfg();

      duplicates_t dups = collect_duplicates(code);
      if (dups.size() > 0) {
        record_stats(dups);
        deduplicate(dups, method);
      }
    });
    report_stats();
  }

 private:
  using duplicates_t =
      std::unordered_map<BlockAsKey, std::unordered_set<Block*>, BlockHasher>;
  const char* METRIC_BLOCKS_REMOVED = "blocks_removed";
  const char* METRIC_ELIGIBLE_BLOCKS = "eligible_blocks";
  const std::vector<DexClass*>& m_scope;
  PassManager& m_mgr;
  const DedupBlocksPass::Config& m_config;

  // map from block size to number of blocks with that size
  std::unordered_map<size_t, size_t> m_dup_sizes;

  // Find blocks with the same exact code
  duplicates_t collect_duplicates(IRCode* code) {
    const auto& blocks = code->cfg().blocks();
    duplicates_t duplicates;

    int num_eligible_blocks = 0;
    for (Block* block : blocks) {
      if (has_opcodes(block) && should_remove(code->cfg(), block)) {
        duplicates[BlockAsKey{code, block}].insert(block);
        ++num_eligible_blocks;
      }
    }
    m_mgr.incr_metric(METRIC_ELIGIBLE_BLOCKS, num_eligible_blocks);

    remove_singletons(duplicates);
    // the hash function isn't perfect, so check behind with an equals function
    return duplicates;
  }

  // remove all but one of a duplicate set. Reroute the predecessors to the
  // canonical block
  void deduplicate(const duplicates_t& dups, DexMethod* method) {
    const auto& code = method->get_code();
    for (const auto& entry : dups) {
      std::unordered_set<Block*> blocks = entry.second;

      // canon is block with lowest id.
      Block* canon = nullptr;
      size_t canon_id = std::numeric_limits<size_t>::max();
      for (Block* block : blocks) {
        size_t id = block->id();
        if (id <= canon_id) {
          canon_id = id;
          canon = block;
        }
      }

      for (Block* block : blocks) {
        if (block->id() == canon_id) {
          // We remove the debug line information because it will be incorrect
          // for every block we reroute here. When there's no debug info,
          // the jvm will report the error on the closing brace of the function.
          // It's not perfect but it's better than incorrect information.
          code->remove_debug_line_info(canon);
        } else {
          transform::replace_block(code, block, canon);
          m_mgr.incr_metric(METRIC_BLOCKS_REMOVED, 1);
        }
      }
    }
  }

  static bool should_remove(const ControlFlowGraph& cfg, Block* block) {
    if (!has_opcodes(block)) {
      return false;
    }

    if (is_catch(block)) {
      // TODO. Should be possible. Skip now for simplicity
      return false;
    }

    // Deal with a verification error like this
    //
    // A: new-instance v0
    //    add-int v1, v2, v3       (this is here to clarify that A != C)
    // B: v0 <init>
    //
    //    ...
    //
    // C: new-instance v0
    // D: v0 <init>
    //
    // B == D. Coalesce!
    //
    // A: new-instance v0
    //    add-int v1, v2, v3
    // B: v0 <init>
    //
    // C: new-instance v0
    //    goto B
    //
    // But the verifier doesn't like this. When it merges v0 on B,
    // it declares it to be a conflict because they were instantiated
    // on different lines.
    // See androidxref.com/6.0.1_r10/xref/art/runtime/verifier/reg_type.cc#684
    //
    // It would be impossible to write this in java, but if you tried it would
    // look like this
    //
    // if (someCondition) {
    //   Foo a;
    // } else {
    //   Foo b;
    // }
    // (a or b) = new Foo();
    //
    // We try to avoid this situation by not considering blocks that call
    // constructors, but this isn't bulletproof. FIXME
    if (calls_constructor(block)) {
      return false;
    }

    // We can't replace blocks that are involved in a fallthrough because they
    // depend on their position in the list of instructions. Deduplicating
    // will involve sending control to a block in a different place.
    for (Block* pred : block->preds()) {
      if (!has_opcodes(pred)) {
        // Skip this case because it's complicated. It should be fixed by
        // upcoming changes to the CFG
        return false;
      }

      if (is_fallthrough(cfg, pred, block)) {
        return false;
      }
    }

    for (Block* succ : block->succs()) {
      if (is_fallthrough(cfg, block, succ)) {
        return false;
      }
    }

    return true;
  }

  static bool calls_constructor(Block* block) {
    for (const auto& mie : InstructionIterable(block)) {
      if (is_invoke(mie.insn->opcode())) {
        auto meth =
            resolve_method(mie.insn->get_method(), opcode_to_search(mie.insn));
        if (meth != nullptr && is_init(meth)) {
          return true;
        }
      }
    }
    return false;
  }

  // `pred` falls through to `succ` iff their connecting edge in the CFG is a
  // goto but `pred`'s last instruction isn't a goto
  static bool is_fallthrough(const ControlFlowGraph& cfg,
                             Block* pred,
                             Block* succ) {
    always_assert_log(has_opcodes(pred), "need opcodes");
    const auto& flags = cfg.edge(pred, succ);
    const auto& last_of_pred = last_opcode(pred);

    return flags[EDGE_GOTO] && !is_goto(last_of_pred->insn->opcode());
  }

  void record_stats(const duplicates_t& duplicates) {
    for (const auto& entry : duplicates) {
      bool first = true;
      bool same_size = true;
      size_t size = 0;
      for (Block* block : entry.second) {
        size_t this_size = num_opcodes(block);
        if (first) {
          size = this_size;
          first = false;
        } else if (size != this_size) {
          same_size = false;
        }
      }
      if (same_size && size > 0) {
        m_dup_sizes[size] += entry.second.size();
      }
    }
  }

  void report_stats() {
    TRACE(DEDUP_BLOCKS,
          2,
          "%d eligible_blocks\n",
          m_mgr.get_metric(METRIC_ELIGIBLE_BLOCKS));

    for (const auto& entry : m_dup_sizes) {
      TRACE(DEDUP_BLOCKS,
            2,
            "found %d duplicate blocks with %d instructions\n",
            entry.second,
            entry.first);
    }

    TRACE(DEDUP_BLOCKS,
          1,
          "%d blocks removed\n",
          m_mgr.get_metric(METRIC_BLOCKS_REMOVED));
  }

  // remove sets with only one block
  static void remove_singletons(duplicates_t& duplicates) {
    for (auto it = duplicates.begin(); it != duplicates.end();) {
      if (it->second.size() <= 1) {
        it = duplicates.erase(it);
      } else {
        ++it;
      }
    }
  }

  static boost::optional<MethodItemEntry&> last_opcode(Block* block) {
    for (auto it = block->rbegin(); it != block->rend(); it++) {
      if (it->type == MFLOW_OPCODE) {
        return *it;
      }
    }
    return boost::none;
  }

  static bool has_opcodes(Block* block) {
    const auto& it = InstructionIterable(block);
    return it.begin() != it.end();
  }

  static size_t num_opcodes(Block* block) {
    size_t result = 0;
    const auto& iterable = InstructionIterable(block);
    for (auto it = iterable.begin(); it != iterable.end(); it++) {
      result++;
    }
    return result;
  }

  static void print_dups(duplicates_t dups) {
    TRACE(DEDUP_BLOCKS, 4, "duplicate blocks set: {\n");
    for (const auto& entry : dups) {
      TRACE(DEDUP_BLOCKS, 4, "  hash = %lu\n", BlockHasher{}(entry.first));
      for (Block* b : entry.second) {
        TRACE(DEDUP_BLOCKS, 4, "    block %d\n", b->id());
        for (const MethodItemEntry& mie : *b) {
          TRACE(DEDUP_BLOCKS, 4, "      %s\n", SHOW(mie));
        }
      }
    }
    TRACE(DEDUP_BLOCKS, 4, "} end duplicate blocks set\n");
  }
};
}

void DedupBlocksPass::run_pass(DexStoresVector& stores,
                               ConfigFiles& /* unused */,
                               PassManager& mgr) {
  const auto& scope = build_class_scope(stores);
  DedupBlocksImpl impl(scope, mgr, m_config);
  impl.run();
}

static DedupBlocksPass s_pass;
<commit_msg>Parallelize DedupBlocksPass<commit_after>/**
 * Copyright (c) 2016-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

#include "DedupBlocksPass.h"

#include <atomic>
#include <boost/optional.hpp>
#include <iterator>
#include <mutex>
#include <unordered_map>
#include <unordered_set>

#include "ControlFlow.h"
#include "DexClass.h"
#include "DexOutput.h"
#include "DexUtil.h"
#include "IRCode.h"
#include "ParallelWalkers.h"
#include "Resolver.h"
#include "Transform.h"

/*
 * This pass removes blocks that are duplicates in a method.
 *
 * If a method has multiple blocks with the same code and the same successors,
 * delete all but one of the blocks. Naming one of them the canonical block.
 *
 * Then, reroute all the predecessors of all the blocks to that canonical block.
 */

namespace {

using hash_t = std::size_t;

struct BlockAsKey {
  IRCode* code;
  Block* block;

  BlockAsKey(IRCode* c, Block* b) : code(c), block(b) {}

  bool operator==(const BlockAsKey& other) const {
    return same_successors(other) && same_try_regions(other) &&
           same_code(other);
  }

  // Structural equality of opcodes except branch targets are ignored
  // because they are unknown until we sync back to DexInstructions.
  bool same_code(const BlockAsKey& other) const {
    return InstructionIterable(block).structural_equals(
        InstructionIterable(other.block));
  }

  // The blocks must also have the exact same successors
  // (and reached in the same ways)
  bool same_successors(const BlockAsKey& other) const {
    const auto& b1_succs = this->block->succs();
    const auto& b2_succs = other.block->succs();
    if (b1_succs.size() != b2_succs.size()) {
      return false;
    }
    for (Block* b1_succ : b1_succs) {
      const auto& in_b2 = std::find(b2_succs.begin(), b2_succs.end(), b1_succ);
      if (in_b2 == b2_succs.end()) {
        // b1 has a succ that b2 doesn't
        return false;
      }

      if (code->cfg().edge(this->block, b1_succ) !=
          code->cfg().edge(other.block, *in_b2)) {
        // successors are the same but the type of connecting
        // edges are different.
        // Like so:
        //
        // B0: if-eqz v0, B3
        // B1: return
        // B2: if eqz v0, B1
        // B3: add-int v1, 1
        //
        // B0's succs are B1 (fallthru) and B3 (branch)
        // B2's succs are B3 (fallthru) and B1 (branch)
        return false;
      }
    }
    return true;
  }

  // return true if in the same try region (or both in no try region at all)
  bool same_try_regions(const BlockAsKey& other) const {
    return transform::find_active_catch(code, this->block->begin()) ==
           transform::find_active_catch(code, other.block->begin());
  }
};

struct BlockHasher {
  hash_t operator()(const BlockAsKey& key) const {
    hash_t result = 0;
    for (auto& mie : InstructionIterable(key.block)) {
      result ^= mie.insn->hash();
    }
    return result;
  }
};

class DedupBlocksImpl {
 public:
  DedupBlocksImpl(const std::vector<DexClass*>& scope,
                  PassManager& mgr,
                  const DedupBlocksPass::Config& config)
      : m_scope(scope), m_mgr(mgr), m_config(config) {}

  void run() {
    walk_methods_parallel_simple(m_scope, [this](DexMethod* method) {
      if (m_config.method_black_list.count(method) != 0) {
        return;
      }

      IRCode* code = method->get_code();
      if (code == nullptr) {
        return;
      }
      code->build_cfg();

      duplicates_t dups = collect_duplicates(code);
      if (dups.size() > 0) {
        record_stats(dups);
        deduplicate(dups, method);
      }
    });
    report_stats();
  }

 private:
  using duplicates_t =
      std::unordered_map<BlockAsKey, std::unordered_set<Block*>, BlockHasher>;
  const char* METRIC_BLOCKS_REMOVED = "blocks_removed";
  const char* METRIC_ELIGIBLE_BLOCKS = "eligible_blocks";
  const std::vector<DexClass*>& m_scope;
  PassManager& m_mgr;
  const DedupBlocksPass::Config& m_config;

  // Stats
  std::mutex lock;
  std::atomic_int m_num_eligible_blocks{0};
  std::atomic_int m_num_blocks_removed{0};
  // map from block size to number of blocks with that size
  std::unordered_map<size_t, size_t> m_dup_sizes;

  // Find blocks with the same exact code
  duplicates_t collect_duplicates(IRCode* code) {
    const auto& blocks = code->cfg().blocks();
    duplicates_t duplicates;

    for (Block* block : blocks) {
      if (should_remove(code->cfg(), block)) {
        duplicates[BlockAsKey{code, block}].insert(block);
        ++m_num_eligible_blocks;
      }
    }

    remove_singletons(duplicates);
    return duplicates;
  }

  // remove all but one of a duplicate set. Reroute the predecessors to the
  // canonical block
  void deduplicate(const duplicates_t& dups, DexMethod* method) {
    const auto& code = method->get_code();
    for (const auto& entry : dups) {
      std::unordered_set<Block*> blocks = entry.second;

      // canon is block with lowest id.
      Block* canon = nullptr;
      size_t canon_id = std::numeric_limits<size_t>::max();
      for (Block* block : blocks) {
        size_t id = block->id();
        if (id <= canon_id) {
          canon_id = id;
          canon = block;
        }
      }

      for (Block* block : blocks) {
        if (block->id() == canon_id) {
          // We remove the debug line information because it will be incorrect
          // for every block we reroute here. When there's no debug info,
          // the jvm will report the error on the closing brace of the function.
          // It's not perfect but it's better than incorrect information.
          code->remove_debug_line_info(canon);
        } else {
          transform::replace_block(code, block, canon);
          ++m_num_blocks_removed;
        }
      }
    }
  }

  static bool should_remove(const ControlFlowGraph& cfg, Block* block) {
    if (!has_opcodes(block)) {
      return false;
    }

    if (is_catch(block)) {
      // TODO. Should be possible. Skip now for simplicity
      return false;
    }

    // Deal with a verification error like this
    //
    // A: new-instance v0
    //    add-int v1, v2, v3       (this is here to clarify that A != C)
    // B: v0 <init>
    //
    //    ...
    //
    // C: new-instance v0
    // D: v0 <init>
    //
    // B == D. Coalesce!
    //
    // A: new-instance v0
    //    add-int v1, v2, v3
    // B: v0 <init>
    //
    // C: new-instance v0
    //    goto B
    //
    // But the verifier doesn't like this. When it merges v0 on B,
    // it declares it to be a conflict because they were instantiated
    // on different lines.
    // See androidxref.com/6.0.1_r10/xref/art/runtime/verifier/reg_type.cc#684
    //
    // It would be impossible to write this in java, but if you tried it would
    // look like this
    //
    // if (someCondition) {
    //   Foo a;
    // } else {
    //   Foo b;
    // }
    // (a or b) = new Foo();
    //
    // We try to avoid this situation by not considering blocks that call
    // constructors, but this isn't bulletproof. FIXME
    if (calls_constructor(block)) {
      return false;
    }

    // We can't replace blocks that are involved in a fallthrough because they
    // depend on their position in the list of instructions. Deduplicating
    // will involve sending control to a block in a different place.
    for (Block* pred : block->preds()) {
      if (!has_opcodes(pred)) {
        // Skip this case because it's complicated. It should be fixed by
        // upcoming changes to the CFG
        return false;
      }

      if (is_fallthrough(cfg, pred, block)) {
        return false;
      }
    }

    for (Block* succ : block->succs()) {
      if (is_fallthrough(cfg, block, succ)) {
        return false;
      }
    }

    return true;
  }

  static bool calls_constructor(Block* block) {
    for (const auto& mie : InstructionIterable(block)) {
      if (is_invoke(mie.insn->opcode())) {
        auto meth =
            resolve_method(mie.insn->get_method(), opcode_to_search(mie.insn));
        if (meth != nullptr && is_init(meth)) {
          return true;
        }
      }
    }
    return false;
  }

  // `pred` falls through to `succ` iff their connecting edge in the CFG is a
  // goto but `pred`'s last instruction isn't a goto
  static bool is_fallthrough(const ControlFlowGraph& cfg,
                             Block* pred,
                             Block* succ) {
    always_assert_log(has_opcodes(pred), "need opcodes");
    const auto& flags = cfg.edge(pred, succ);
    const auto& last_of_pred = last_opcode(pred);

    return flags[EDGE_GOTO] && !is_goto(last_of_pred->insn->opcode());
  }

  void record_stats(const duplicates_t& duplicates) {
    std::lock_guard<std::mutex> guard{lock};
    for (const auto& entry : duplicates) {
      std::unordered_set<Block*> blocks = entry.second;
      // all blocks have the same number of opcodes
      Block* block = *blocks.begin();
      m_dup_sizes[num_opcodes(block)] += blocks.size();
    }
  }

  void report_stats() {
    int eligible_blocks = m_num_eligible_blocks.load();
    int removed = m_num_blocks_removed.load();
    m_mgr.incr_metric(METRIC_ELIGIBLE_BLOCKS, eligible_blocks);
    m_mgr.incr_metric(METRIC_BLOCKS_REMOVED, removed);
    TRACE(DEDUP_BLOCKS, 2, "%d eligible_blocks\n", eligible_blocks);

    for (const auto& entry : m_dup_sizes) {
      TRACE(DEDUP_BLOCKS,
            2,
            "found %d duplicate blocks with %d instructions\n",
            entry.second,
            entry.first);
    }

    TRACE(DEDUP_BLOCKS, 1, "%d blocks removed\n", removed);
  }

  // remove sets with only one block
  static void remove_singletons(duplicates_t& duplicates) {
    for (auto it = duplicates.begin(); it != duplicates.end();) {
      if (it->second.size() <= 1) {
        it = duplicates.erase(it);
      } else {
        ++it;
      }
    }
  }

  static boost::optional<MethodItemEntry&> last_opcode(Block* block) {
    for (auto it = block->rbegin(); it != block->rend(); it++) {
      if (it->type == MFLOW_OPCODE) {
        return *it;
      }
    }
    return boost::none;
  }

  static bool has_opcodes(Block* block) {
    const auto& it = InstructionIterable(block);
    return it.begin() != it.end();
  }

  static size_t num_opcodes(Block* block) {
    size_t result = 0;
    const auto& iterable = InstructionIterable(block);
    for (auto it = iterable.begin(); it != iterable.end(); it++) {
      result++;
    }
    return result;
  }

  static void print_dups(duplicates_t dups) {
    TRACE(DEDUP_BLOCKS, 4, "duplicate blocks set: {\n");
    for (const auto& entry : dups) {
      TRACE(DEDUP_BLOCKS, 4, "  hash = %lu\n", BlockHasher{}(entry.first));
      for (Block* b : entry.second) {
        TRACE(DEDUP_BLOCKS, 4, "    block %d\n", b->id());
        for (const MethodItemEntry& mie : *b) {
          TRACE(DEDUP_BLOCKS, 4, "      %s\n", SHOW(mie));
        }
      }
    }
    TRACE(DEDUP_BLOCKS, 4, "} end duplicate blocks set\n");
  }
};
}

void DedupBlocksPass::run_pass(DexStoresVector& stores,
                               ConfigFiles& /* unused */,
                               PassManager& mgr) {
  const auto& scope = build_class_scope(stores);
  DedupBlocksImpl impl(scope, mgr, m_config);
  impl.run();
}

static DedupBlocksPass s_pass;
<|endoftext|>


<commit_before>#include "simplify_instructions.hpp"
#include "analyzation.hpp"
#include "local_constant_tracker.hpp"
#include <p0i/emitter.hpp>
#include <p0run/value.hpp>
#include <boost/foreach.hpp>
#include <boost/optional.hpp>
#include <unordered_map>

namespace p0
{
	p0::intermediate::function::instruction_vector simplify_instructions(p0::intermediate::function::instruction_vector const &original)
	{
		p0::intermediate::function::instruction_vector optimized = original;
		local_constant_tracker known_locals;
		auto const jump_destinations = find_all_jump_destinations(original);
		for (std::size_t i = 0; i < optimized.size(); ++i)
		{
			if (jump_destinations.find(i) != jump_destinations.end())
			{
				known_locals.handle_incoming_jump();
			}

			auto &instruction = optimized[i];

			switch (instruction.type())
			{
			case p0::intermediate::instruction_type::set_from_constant:
				{
					known_locals.update_value(static_cast<local_address>(instruction.arguments()[0]), instruction.arguments()[1]);
					break;
				}

			case p0::intermediate::instruction_type::copy:
				{
					auto const source = static_cast<local_address>(instruction.arguments()[1]);
					auto const value = known_locals.find_current_value(source);
					if (value)
					{
						auto const destination = static_cast<local_address>(instruction.arguments()[0]);
						instruction = intermediate::make_set_from_constant(destination, *value);
					}
					break;
				}

			case p0::intermediate::instruction_type::jump_if:
			case p0::intermediate::instruction_type::jump_if_not:
				{
					auto const condition = static_cast<local_address>(instruction.arguments()[1]);
					auto const value = known_locals.find_current_value(condition);
					if (value)
					{
						auto const destination = instruction.arguments()[0];
						bool const value_for_jump = (instruction.type() == p0::intermediate::instruction_type::jump_if);
						if (run::to_boolean(run::value(*value)) == value_for_jump)
						{
							instruction = intermediate::instruction(intermediate::instruction_type::jump, destination);
						}
						else
						{
							instruction = intermediate::instruction(intermediate::instruction_type::nothing);
						}
					}
					break;
				}

			default:
				break;
			}
		}
		return optimized;
	}
}
<commit_msg>fix copy in simplify_instructions<commit_after>#include "simplify_instructions.hpp"
#include "analyzation.hpp"
#include "local_constant_tracker.hpp"
#include <p0i/emitter.hpp>
#include <p0run/value.hpp>
#include <boost/foreach.hpp>
#include <boost/optional.hpp>
#include <unordered_map>

namespace p0
{
	p0::intermediate::function::instruction_vector simplify_instructions(p0::intermediate::function::instruction_vector const &original)
	{
		p0::intermediate::function::instruction_vector optimized = original;
		local_constant_tracker known_locals;
		auto const jump_destinations = find_all_jump_destinations(original);
		for (std::size_t i = 0; i < optimized.size(); ++i)
		{
			if (jump_destinations.find(i) != jump_destinations.end())
			{
				known_locals.handle_incoming_jump();
			}

			auto &instruction = optimized[i];

			switch (instruction.type())
			{
			case p0::intermediate::instruction_type::set_from_constant:
				{
					known_locals.update_value(static_cast<local_address>(instruction.arguments()[0]), instruction.arguments()[1]);
					break;
				}

			case p0::intermediate::instruction_type::copy:
				{
					auto const destination = static_cast<local_address>(instruction.arguments()[0]);
					auto const source = static_cast<local_address>(instruction.arguments()[1]);
					auto const value = known_locals.find_current_value(source);
					if (value)
					{
						instruction = intermediate::make_set_from_constant(destination, *value);
						known_locals.update_value(destination, *value);
					}
					else
					{
						known_locals.set_unknown(destination);
					}
					break;
				}

			case p0::intermediate::instruction_type::jump_if:
			case p0::intermediate::instruction_type::jump_if_not:
				{
					auto const condition = static_cast<local_address>(instruction.arguments()[1]);
					auto const value = known_locals.find_current_value(condition);
					if (value)
					{
						auto const destination = instruction.arguments()[0];
						bool const value_for_jump = (instruction.type() == p0::intermediate::instruction_type::jump_if);
						if (run::to_boolean(run::value(*value)) == value_for_jump)
						{
							instruction = intermediate::instruction(intermediate::instruction_type::jump, destination);
						}
						else
						{
							instruction = intermediate::instruction(intermediate::instruction_type::nothing);
						}
					}
					break;
				}

			default:
				break;
			}
		}
		return optimized;
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2013-present Barefoot Networks, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "action.h"

namespace BMV2 {

void
ConvertActions::convertActionBody(const IR::Vector<IR::StatOrDecl>* body, Util::JsonArray* result) {
    for (auto s : *body) {
        // TODO(jafingerhut) - add line/col at all individual cases below,
        // or perhaps it can be done as a common case above or below
        // for all of them?
        if (!s->is<IR::Statement>()) {
            continue;
        } else if (auto block = s->to<IR::BlockStatement>()) {
            convertActionBody(&block->components, result);
            continue;
        } else if (s->is<IR::ReturnStatement>()) {
            break;
        } else if (s->is<IR::ExitStatement>()) {
            auto primitive = mkPrimitive("exit", result);
            (void)mkParameters(primitive);
            primitive->emplace_non_null("source_info", s->sourceInfoJsonObj());
            break;
        } else if (s->is<IR::AssignmentStatement>()) {
            const IR::Expression* l, *r;
            auto assign = s->to<IR::AssignmentStatement>();
            l = assign->left;
            r = assign->right;

            cstring operation;
            auto type = typeMap->getType(l, true);
            if (type->is<IR::Type_StructLike>())
                operation = "assign_header";
            else if (type->is<IR::Type_Varbits>())
                operation = "assign_VL";
            else if (type->is<IR::Type_HeaderUnion>())
                operation = "assign_union";
            else
                operation = "assign";
            auto primitive = mkPrimitive(operation, result);
            auto parameters = mkParameters(primitive);
            primitive->emplace_non_null("source_info", assign->sourceInfoJsonObj());
            auto left = conv->convertLeftValue(l);
            parameters->append(left);
            bool convertBool = type->is<IR::Type_Boolean>();
            auto right = conv->convert(r, true, true, convertBool);
            parameters->append(right);
            continue;
        } else if (s->is<IR::EmptyStatement>()) {
            continue;
        } else if (s->is<IR::MethodCallStatement>()) {
            LOG3("Visit " << dbp(s));
            auto mc = s->to<IR::MethodCallStatement>()->methodCall;
            auto mi = P4::MethodInstance::resolve(mc, refMap, typeMap);
            if (mi->is<P4::ActionCall>()) {
                BUG("%1%: action call should have been inlined", mc);
                continue;
            } else if (mi->is<P4::BuiltInMethod>()) {
                auto builtin = mi->to<P4::BuiltInMethod>();

                cstring prim;
                auto parameters = new Util::JsonArray();
                auto obj = conv->convert(builtin->appliedTo);
                parameters->append(obj);

                if (builtin->name == IR::Type_Header::setValid) {
                    prim = "add_header";
                } else if (builtin->name == IR::Type_Header::setInvalid) {
                    prim = "remove_header";
                } else if (builtin->name == IR::Type_Stack::push_front) {
                    BUG_CHECK(mc->arguments->size() == 1, "Expected 1 argument for %1%", mc);
                    auto arg = conv->convert(mc->arguments->at(0));
                    prim = "push";
                    parameters->append(arg);
                } else if (builtin->name == IR::Type_Stack::pop_front) {
                    BUG_CHECK(mc->arguments->size() == 1, "Expected 1 argument for %1%", mc);
                    auto arg = conv->convert(mc->arguments->at(0));
                    prim = "pop";
                    parameters->append(arg);
                } else {
                    BUG("%1%: Unexpected built-in method", s);
                }
                auto primitive = mkPrimitive(prim, result);
                primitive->emplace("parameters", parameters);
                primitive->emplace_non_null("source_info", s->sourceInfoJsonObj());
                continue;
            } else if (mi->is<P4::ExternMethod>()) {
                auto em = mi->to<P4::ExternMethod>();
                LOG3("P4V1:: convert " << s);
                backend->getSimpleSwitch()->convertExternObjects(result, em, mc, s);
                continue;
            } else if (mi->is<P4::ExternFunction>()) {
                auto ef = mi->to<P4::ExternFunction>();
                backend->getSimpleSwitch()->convertExternFunctions(result, ef, mc, s);
                continue;
            }
        }
        ::error("%1%: not yet supported on this target", s);
    }
}

void
ConvertActions::convertActionParams(const IR::ParameterList *parameters, Util::JsonArray* params) {
    for (auto p : *parameters->getEnumerator()) {
        if (!refMap->isUsed(p))
            ::warning("Unused action parameter %1%", p);

        auto param = new Util::JsonObject();
        param->emplace("name", p->name);
        auto type = typeMap->getType(p, true);
        if (!type->is<IR::Type_Bits>())
            ::error("%1%: Action parameters can only be bit<> or int<> on this target", p);
        param->emplace("bitwidth", type->width_bits());
        params->append(param);
    }
}

void
ConvertActions::createActions() {
    for (auto it : backend->getStructure().actions) {
        auto action = it.first;
        cstring name = extVisibleName(action);
        auto params = new Util::JsonArray();
        convertActionParams(action->parameters, params);
        auto body = new Util::JsonArray();
        convertActionBody(&action->body->components, body);
        auto id = json->add_action(name, params, body);
        backend->getStructure().ids.emplace(action, id);
    }
}

bool
ConvertActions::preorder(const IR::PackageBlock*) {
    createActions();
    return false;
}

}  // namespace BMV2
<commit_msg>Fix from code review<commit_after>/*
Copyright 2013-present Barefoot Networks, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "action.h"

namespace BMV2 {

void
ConvertActions::convertActionBody(const IR::Vector<IR::StatOrDecl>* body, Util::JsonArray* result) {
    for (auto s : *body) {
        // TODO(jafingerhut) - add line/col at all individual cases below,
        // or perhaps it can be done as a common case above or below
        // for all of them?
        if (!s->is<IR::Statement>()) {
            continue;
        } else if (auto block = s->to<IR::BlockStatement>()) {
            convertActionBody(&block->components, result);
            continue;
        } else if (s->is<IR::ReturnStatement>()) {
            break;
        } else if (s->is<IR::ExitStatement>()) {
            auto primitive = mkPrimitive("exit", result);
            (void)mkParameters(primitive);
            primitive->emplace_non_null("source_info", s->sourceInfoJsonObj());
            break;
        } else if (s->is<IR::AssignmentStatement>()) {
            const IR::Expression* l, *r;
            auto assign = s->to<IR::AssignmentStatement>();
            l = assign->left;
            r = assign->right;

            cstring operation;
            auto type = typeMap->getType(l, true);
            if (type->is<IR::Type_Varbits>())
                operation = "assign_VL";
            else if (type->is<IR::Type_HeaderUnion>())
                operation = "assign_union";
            else if (type->is<IR::Type_StructLike>())
                operation = "assign_header";
            else
                operation = "assign";
            auto primitive = mkPrimitive(operation, result);
            auto parameters = mkParameters(primitive);
            primitive->emplace_non_null("source_info", assign->sourceInfoJsonObj());
            auto left = conv->convertLeftValue(l);
            parameters->append(left);
            bool convertBool = type->is<IR::Type_Boolean>();
            auto right = conv->convert(r, true, true, convertBool);
            parameters->append(right);
            continue;
        } else if (s->is<IR::EmptyStatement>()) {
            continue;
        } else if (s->is<IR::MethodCallStatement>()) {
            LOG3("Visit " << dbp(s));
            auto mc = s->to<IR::MethodCallStatement>()->methodCall;
            auto mi = P4::MethodInstance::resolve(mc, refMap, typeMap);
            if (mi->is<P4::ActionCall>()) {
                BUG("%1%: action call should have been inlined", mc);
                continue;
            } else if (mi->is<P4::BuiltInMethod>()) {
                auto builtin = mi->to<P4::BuiltInMethod>();

                cstring prim;
                auto parameters = new Util::JsonArray();
                auto obj = conv->convert(builtin->appliedTo);
                parameters->append(obj);

                if (builtin->name == IR::Type_Header::setValid) {
                    prim = "add_header";
                } else if (builtin->name == IR::Type_Header::setInvalid) {
                    prim = "remove_header";
                } else if (builtin->name == IR::Type_Stack::push_front) {
                    BUG_CHECK(mc->arguments->size() == 1, "Expected 1 argument for %1%", mc);
                    auto arg = conv->convert(mc->arguments->at(0));
                    prim = "push";
                    parameters->append(arg);
                } else if (builtin->name == IR::Type_Stack::pop_front) {
                    BUG_CHECK(mc->arguments->size() == 1, "Expected 1 argument for %1%", mc);
                    auto arg = conv->convert(mc->arguments->at(0));
                    prim = "pop";
                    parameters->append(arg);
                } else {
                    BUG("%1%: Unexpected built-in method", s);
                }
                auto primitive = mkPrimitive(prim, result);
                primitive->emplace("parameters", parameters);
                primitive->emplace_non_null("source_info", s->sourceInfoJsonObj());
                continue;
            } else if (mi->is<P4::ExternMethod>()) {
                auto em = mi->to<P4::ExternMethod>();
                LOG3("P4V1:: convert " << s);
                backend->getSimpleSwitch()->convertExternObjects(result, em, mc, s);
                continue;
            } else if (mi->is<P4::ExternFunction>()) {
                auto ef = mi->to<P4::ExternFunction>();
                backend->getSimpleSwitch()->convertExternFunctions(result, ef, mc, s);
                continue;
            }
        }
        ::error("%1%: not yet supported on this target", s);
    }
}

void
ConvertActions::convertActionParams(const IR::ParameterList *parameters, Util::JsonArray* params) {
    for (auto p : *parameters->getEnumerator()) {
        if (!refMap->isUsed(p))
            ::warning("Unused action parameter %1%", p);

        auto param = new Util::JsonObject();
        param->emplace("name", p->name);
        auto type = typeMap->getType(p, true);
        if (!type->is<IR::Type_Bits>())
            ::error("%1%: Action parameters can only be bit<> or int<> on this target", p);
        param->emplace("bitwidth", type->width_bits());
        params->append(param);
    }
}

void
ConvertActions::createActions() {
    for (auto it : backend->getStructure().actions) {
        auto action = it.first;
        cstring name = extVisibleName(action);
        auto params = new Util::JsonArray();
        convertActionParams(action->parameters, params);
        auto body = new Util::JsonArray();
        convertActionBody(&action->body->components, body);
        auto id = json->add_action(name, params, body);
        backend->getStructure().ids.emplace(action, id);
    }
}

bool
ConvertActions::preorder(const IR::PackageBlock*) {
    createActions();
    return false;
}

}  // namespace BMV2
<|endoftext|>


<commit_before>/**
 * @author André Anjos <andre.anjos@idiap.ch>
 * @date Tue Jan 18 17:07:26 2011 +0100
 *
 * @brief Combines all modules to make up the complete bindings
 *
 * Copyright (C) 2011-2014 Idiap Research Institute, Martigny, Switzerland
 */

#include "bob/config.h"
#include "ndarray.h"

/** extra bindings required for compatibility **/
void bind_core_tinyvector();
void bind_core_ndarray_numpy();
void bind_core_bz_numpy();
void bind_sp_extrapolate();
void bind_sp_convolution();

void bind_ip_block();
void bind_ip_crop_shift();
void bind_ip_extrapolate_mask();
void bind_ip_flipflop();
void bind_ip_gamma_correction();
void bind_ip_integral();
void bind_ip_scale();
void bind_ip_shear();
void bind_ip_rotate();
void bind_ip_dctfeatures();
void bind_ip_geomnorm();
void bind_ip_faceeyesnorm();
void bind_ip_tantriggs();
void bind_ip_histogram();
void bind_ip_lbp();
void bind_ip_gaussian();
void bind_ip_gaussian_scale_space();
void bind_ip_wgaussian();
void bind_ip_msr();
void bind_ip_sqi();
void bind_ip_median();
void bind_ip_sobel();
void bind_ip_hog();
void bind_ip_glcm_uint8();
void bind_ip_glcm_uint16();
void bind_ip_glcmprop();
void bind_ip_sift();

#if WITH_VLFEAT
void bind_ip_vlsift();
void bind_ip_vldsift();
#endif

BOOST_PYTHON_MODULE(_old_library) {

  boost::python::docstring_options docopt(true, true, false);

  bob::python::setup_python("old-style bob image processing classes and sub-classes");

  /** extra bindings required for compatibility **/
  bind_core_tinyvector();
  bind_core_ndarray_numpy();
  bind_core_bz_numpy();
  bind_sp_extrapolate();
  bind_sp_convolution();

  bind_ip_block();
  bind_ip_crop_shift();
  bind_ip_extrapolate_mask();
  bind_ip_flipflop();
  bind_ip_gamma_correction();
  bind_ip_integral();
  bind_ip_scale();
  bind_ip_shear();
  bind_ip_rotate();
  bind_ip_dctfeatures();
  bind_ip_geomnorm();
  bind_ip_faceeyesnorm();
  bind_ip_tantriggs();
  bind_ip_histogram();
  bind_ip_lbp();
  bind_ip_gaussian();
  bind_ip_gaussian_scale_space();
  bind_ip_wgaussian();
  bind_ip_msr();
  bind_ip_sqi();
  bind_ip_median();
  bind_ip_sobel();
  bind_ip_hog();
  bind_ip_glcm_uint8();
  bind_ip_glcm_uint16();
  bind_ip_glcmprop();
  bind_ip_sift();

#if WITH_VLFEAT
  bind_ip_vlsift();
  bind_ip_vldsift();
#endif

}
<commit_msg>Update main.cc<commit_after>/**
 * @author André Anjos <andre.anjos@idiap.ch>
 * @date Tue Jan 18 17:07:26 2011 +0100
 *
 * @brief Combines all modules to make up the complete bindings
 *
 * Copyright (C) 2011-2014 Idiap Research Institute, Martigny, Switzerland
 */

#include "bob/config.h"
#include "ndarray.h"

/** extra bindings required for compatibility **/
void bind_core_tinyvector();
void bind_core_ndarray_numpy();
void bind_core_bz_numpy();
void bind_sp_extrapolate();
void bind_sp_convolution();

void bind_ip_block();
void bind_ip_crop_shift();
void bind_ip_extrapolate_mask();
void bind_ip_flipflop();
void bind_ip_gamma_correction();
void bind_ip_scale();
void bind_ip_shear();
void bind_ip_rotate();
void bind_ip_dctfeatures();
void bind_ip_geomnorm();
void bind_ip_faceeyesnorm();
void bind_ip_tantriggs();
void bind_ip_histogram();
void bind_ip_gaussian();
void bind_ip_gaussian_scale_space();
void bind_ip_wgaussian();
void bind_ip_msr();
void bind_ip_sqi();
void bind_ip_median();
void bind_ip_sobel();
void bind_ip_hog();
void bind_ip_glcm_uint8();
void bind_ip_glcm_uint16();
void bind_ip_glcmprop();
void bind_ip_sift();

#if WITH_VLFEAT
void bind_ip_vlsift();
void bind_ip_vldsift();
#endif

BOOST_PYTHON_MODULE(_old_library) {

  boost::python::docstring_options docopt(true, true, false);

  bob::python::setup_python("old-style bob image processing classes and sub-classes");

  /** extra bindings required for compatibility **/
  bind_core_tinyvector();
  bind_core_ndarray_numpy();
  bind_core_bz_numpy();
  bind_sp_extrapolate();
  bind_sp_convolution();

  bind_ip_block();
  bind_ip_crop_shift();
  bind_ip_extrapolate_mask();
  bind_ip_flipflop();
  bind_ip_gamma_correction();
  bind_ip_scale();
  bind_ip_shear();
  bind_ip_rotate();
  bind_ip_dctfeatures();
  bind_ip_geomnorm();
  bind_ip_faceeyesnorm();
  bind_ip_tantriggs();
  bind_ip_histogram();
  bind_ip_gaussian();
  bind_ip_gaussian_scale_space();
  bind_ip_wgaussian();
  bind_ip_msr();
  bind_ip_sqi();
  bind_ip_median();
  bind_ip_sobel();
  bind_ip_hog();
  bind_ip_glcm_uint8();
  bind_ip_glcm_uint16();
  bind_ip_glcmprop();
  bind_ip_sift();

#if WITH_VLFEAT
  bind_ip_vlsift();
  bind_ip_vldsift();
#endif

}
<|endoftext|>


<commit_before>#include "flaggedarrayset.h"

#include <map>
#include <set>
#include <vector>
#include <list>
#include <thread>
#include <mutex>
#include <string.h>
#include <assert.h>
#include <stdio.h>

/******************************
 **** FlaggedArraySet util ****
 ******************************/
struct PtrPair {
	std::shared_ptr<std::vector<unsigned char> > elem;
	std::shared_ptr<std::vector<unsigned char> > elemHash;
	PtrPair(const std::shared_ptr<std::vector<unsigned char> >& elemIn, const std::shared_ptr<std::vector<unsigned char> >& elemHashIn) :
		elem(elemIn), elemHash(elemHashIn) {}
};

struct SharedPtrElem {
	PtrPair e;
	bool operator==(const SharedPtrElem& o) const { return *e.elemHash == *o.e.elemHash; }
	bool operator!=(const SharedPtrElem& o) const { return *e.elemHash != *o.e.elemHash; }
	bool operator< (const SharedPtrElem& o) const { return *e.elemHash <  *o.e.elemHash; }
	bool operator<=(const SharedPtrElem& o) const { return *e.elemHash <= *o.e.elemHash; }
	bool operator> (const SharedPtrElem& o) const { return *e.elemHash >  *o.e.elemHash; }
	bool operator>=(const SharedPtrElem& o) const { return *e.elemHash >= *o.e.elemHash; }
	SharedPtrElem(const PtrPair& eIn) : e(eIn) {}
};

class Deduper {
private:
	std::mutex dedup_mutex;
	std::set<FlaggedArraySet*> allArraySets;
	std::thread dedup_thread;
public:
	Deduper()
		: dedup_thread([&]() {
#ifdef PRECISE_BENCH
			return;
#endif
			while (true) {
				bool haveMultipleSets = false;
				{
					std::lock_guard<std::mutex> lock(dedup_mutex);
					haveMultipleSets = allArraySets.size() > 1;
				}

				if (haveMultipleSets) {
					std::list<PtrPair> ptrlist;

					{
						std::lock_guard<std::mutex> lock(dedup_mutex);
						for (FlaggedArraySet* fas : allArraySets) {
							if (fas->allowDups)
								continue;
							if (!fas->mutex.try_lock())
								continue;
							std::lock_guard<WaitCountMutex> lock(fas->mutex, std::adopt_lock);
							for (const auto& e : fas->backingMap) {
								if (fas->mutex.wait_count())
									break;
								assert(e.first.elem);
								ptrlist.push_back(PtrPair(e.first.elem, e.first.elemHash));
							}
						}
					}

					std::set<SharedPtrElem> txset;
					std::map<std::vector<unsigned char>*, PtrPair> duplicateMap;
					std::list<PtrPair> deallocList;
					for (const auto& ptr : ptrlist) {
						assert(ptr.elemHash);
						auto res = txset.insert(SharedPtrElem(ptr));
						if (!res.second && res.first->e.elem != ptr.elem)
							duplicateMap.insert(std::make_pair(&(*ptr.elem), res.first->e));
					}

					int dedups = 0;
					{
						std::lock_guard<std::mutex> lock(dedup_mutex);
						for (FlaggedArraySet* fas : allArraySets) {
							if (fas->allowDups)
								continue;
							if (!fas->mutex.try_lock())
								continue;
							std::lock_guard<WaitCountMutex> lock(fas->mutex, std::adopt_lock);
							for (auto& e : fas->backingMap) {
								if (fas->mutex.wait_count())
									break;
								assert(e.first.elem);
								auto it = duplicateMap.find(&(*e.first.elem));
								if (it != duplicateMap.end()) {
									assert(*it->second.elem == *e.first.elem);
									assert(*it->second.elemHash == *e.first.elemHash);
									deallocList.emplace_back(it->second);
									const_cast<ElemAndFlag&>(e.first).elem.swap(deallocList.back().elem);
									const_cast<ElemAndFlag&>(e.first).elemHash.swap(deallocList.back().elemHash);
									dedups++;
								}
							}
						}
					}
#ifdef FOR_TEST
					if (dedups)
						printf("Deduped %d txn\n", dedups);
#endif
				}
#ifdef FOR_TEST
				std::this_thread::sleep_for(std::chrono::milliseconds(5));
#else
				std::this_thread::sleep_for(std::chrono::milliseconds(5000));
#endif
			}
		})
	{}

	~Deduper() {
		//TODO: close thread
	}

	void addFAS(FlaggedArraySet* fas) {
		std::lock_guard<std::mutex> lock(dedup_mutex);
		allArraySets.insert(fas);
	}

	void removeFAS(FlaggedArraySet* fas) {
		std::lock_guard<std::mutex> lock(dedup_mutex);
		allArraySets.erase(fas);
	}
};

static Deduper* deduper;

FlaggedArraySet::FlaggedArraySet(unsigned int maxSizeIn, bool allowDupsIn) :
		maxSize(maxSizeIn), backingMap(maxSize), allowDups(allowDupsIn) {
	clear();
	if (!deduper)
		deduper = new Deduper();
	deduper->addFAS(this);
}

FlaggedArraySet::~FlaggedArraySet() {
	deduper->removeFAS(this);
	assert(sanity_check());
}

ElemAndFlag::ElemAndFlag(const std::shared_ptr<std::vector<unsigned char> >& elemIn, bool flagIn, bool allowDupsIn, bool setHash) :
	flag(flagIn), allowDups(allowDupsIn), elem(elemIn)
{
	if (setHash) {
		elemHash = std::make_shared<std::vector<unsigned char> >(32);
		double_sha256(&(*elem)[0], &(*elemHash)[0], elem->size());
	}
}
ElemAndFlag::ElemAndFlag(const std::vector<unsigned char>::const_iterator& elemBeginIn, const std::vector<unsigned char>::const_iterator& elemEndIn, bool flagIn, bool allowDupsIn) :
	flag(flagIn), allowDups(allowDupsIn), elemBegin(elemBeginIn), elemEnd(elemEndIn) {}

bool ElemAndFlag::operator == (const ElemAndFlag& o) const {
	if (elem && o.elem) {
		if (allowDups)
			return o.elem == elem;
		bool hashSet = o.elemHash && elemHash;
		return o.elem == elem ||
			(hashSet && *o.elemHash == *elemHash) ||
			(!hashSet && *o.elem == *elem);
	} else {
		std::vector<unsigned char>::const_iterator o_begin, o_end, e_begin, e_end;
		if (elem) {
			e_begin = elem->begin();
			e_end = elem->end();
		} else {
			e_begin = elemBegin;
			e_end = elemEnd;
		}
		if (o.elem) {
			o_begin = o.elem->begin();
			o_end = o.elem->end();
		} else {
			o_begin = o.elemBegin;
			o_end = o.elemEnd;
		}
		return o_end - o_begin == e_end - e_begin && !memcmp(&(*o_begin), &(*e_begin), o_end - o_begin);
	}
}

size_t std::hash<ElemAndFlag>::operator()(const ElemAndFlag& e) const {
	std::vector<unsigned char>::const_iterator it, end;
	if (e.elem) {
		it = e.elem->begin();
		end = e.elem->end();
	} else {
		it = e.elemBegin;
		end = e.elemEnd;
	}

	if (end - it < 5 + 32 + 4) {
		assert(0);
		return 42; // WAT?
	}
	it += 5 + 32 + 4 - 8;
	size_t res = 0;
	static_assert(sizeof(size_t) == 4 || sizeof(size_t) == 8, "Your size_t is neither 32-bit nor 64-bit?");
	for (unsigned int i = 0; i < 8; i += sizeof(size_t)) {
		for (unsigned int j = 0; j < sizeof(size_t); j++)
			res ^= *(it + i + j) << 8*j;
	}
	return res;
}

bool FlaggedArraySet::sanity_check() const {
	size_t size = indexMap.size();
	assert(backingMap.size() == size);
	assert(this->size() == size - to_be_removed.size());

	size_t expected_flag_count = 0;
	for (uint64_t i = 0; i < size; i++) {
		std::unordered_map<ElemAndFlag, uint64_t>::iterator it = indexMap.at(i);
		assert(it != backingMap.end());
		assert(it->second == i + offset);
		assert(backingMap.find(it->first) == it);
		assert(&backingMap.find(it->first)->first == &it->first);
		if (it->first.flag)
			expected_flag_count++;
	}
	assert(expected_flag_count == flag_count);

	ssize_t expected_flags_removed = 0;
	for (int index : to_be_removed) {
		std::unordered_map<ElemAndFlag, uint64_t>::iterator it = indexMap.at(index);
		if (it->first.flag)
			expected_flags_removed++;
	}
	assert(expected_flags_removed == flags_to_remove);

	return expected_flags_removed == flags_to_remove && expected_flag_count == flag_count;
}

void FlaggedArraySet::remove_(size_t index) {
	auto& rm = indexMap[index];
	assert(index < indexMap.size());
	if (rm->first.flag)
		flag_count--;

	size_t size = backingMap.size();

	if (index < size/2) {
		for (uint64_t i = 0; i < index; i++)
			indexMap[i]->second++;
		offset++;
	} else
		for (uint64_t i = index + 1; i < size; i++)
			indexMap[i]->second--;
	backingMap.erase(rm);
	indexMap.erase(indexMap.begin() + index);
}

inline void FlaggedArraySet::cleanup_late_remove() const {
	assert(sanity_check());
	if (to_be_removed.size()) {
		for (unsigned int i = 0; i < to_be_removed.size(); i++) {
			assert((unsigned int)to_be_removed[i] < indexMap.size());
			const_cast<FlaggedArraySet*>(this)->remove_(to_be_removed[i]);
		}
		to_be_removed.clear();
		flags_to_remove = 0;
		max_remove = 0;
	}
	assert(sanity_check());
}

bool FlaggedArraySet::contains(const std::shared_ptr<std::vector<unsigned char> >& e) const {
	std::lock_guard<WaitCountMutex> lock(mutex);
	cleanup_late_remove();
	return backingMap.count(ElemAndFlag(e, false, allowDups, false));
}

void FlaggedArraySet::add(const std::shared_ptr<std::vector<unsigned char> >& e, bool flag) {
	ElemAndFlag elem(e, flag, allowDups, true);

	std::lock_guard<WaitCountMutex> lock(mutex);
	cleanup_late_remove();

	auto res = backingMap.insert(std::make_pair(elem, size() + offset));
	if (!res.second)
		return;

	indexMap.push_back(res.first);

	assert(size() <= maxSize + 1);
	while (size() > maxSize)
		remove_(0);

	if (flag)
		flag_count++;

	assert(sanity_check());
}

int FlaggedArraySet::remove(const std::vector<unsigned char>::const_iterator& start, const std::vector<unsigned char>::const_iterator& end) {
	std::lock_guard<WaitCountMutex> lock(mutex);
	cleanup_late_remove();

	auto it = backingMap.find(ElemAndFlag(start, end, false, allowDups));
	if (it == backingMap.end())
		return -1;

	int res = it->second - offset;
	remove_(res);

	assert(sanity_check());
	return res;
}

bool FlaggedArraySet::remove(int index, std::vector<unsigned char>& elemRes, unsigned char* elemHashRes) {
	std::lock_guard<WaitCountMutex> lock(mutex);

	if (index < max_remove)
		cleanup_late_remove();
	int lookup_index = index + to_be_removed.size();

	if ((unsigned int)lookup_index >= indexMap.size())
		return false;

	const ElemAndFlag& e = indexMap[lookup_index]->first;
	assert(e.elem && e.elemHash);
	memcpy(elemHashRes, &(*e.elemHash)[0], 32);
	elemRes = *e.elem;

	if (index >= max_remove) {
		to_be_removed.push_back(index);
		max_remove = index;
		if (e.flag) flags_to_remove++;
	} else {
		cleanup_late_remove();
		remove_(index);
	}

	assert(sanity_check());
	return true;
}

void FlaggedArraySet::clear() {
	std::lock_guard<WaitCountMutex> lock(mutex);
	if (!indexMap.empty() && !backingMap.empty())
		assert(sanity_check());

	flag_count = 0; offset = 0;
	flags_to_remove = 0; max_remove = 0;
	backingMap.clear(); indexMap.clear(); to_be_removed.clear();
}

void FlaggedArraySet::for_all_txn(const std::function<void (const std::shared_ptr<std::vector<unsigned char> >&)> callback) const {
	std::lock_guard<WaitCountMutex> lock(mutex);
	cleanup_late_remove();
	for (const auto& e : indexMap) {
		assert(e->first.elem);
		callback(e->first.elem);
	}
}
<commit_msg>Fix sanity_check()<commit_after>#include "flaggedarrayset.h"

#include <map>
#include <set>
#include <vector>
#include <list>
#include <thread>
#include <mutex>
#include <string.h>
#include <assert.h>
#include <stdio.h>

/******************************
 **** FlaggedArraySet util ****
 ******************************/
struct PtrPair {
	std::shared_ptr<std::vector<unsigned char> > elem;
	std::shared_ptr<std::vector<unsigned char> > elemHash;
	PtrPair(const std::shared_ptr<std::vector<unsigned char> >& elemIn, const std::shared_ptr<std::vector<unsigned char> >& elemHashIn) :
		elem(elemIn), elemHash(elemHashIn) {}
};

struct SharedPtrElem {
	PtrPair e;
	bool operator==(const SharedPtrElem& o) const { return *e.elemHash == *o.e.elemHash; }
	bool operator!=(const SharedPtrElem& o) const { return *e.elemHash != *o.e.elemHash; }
	bool operator< (const SharedPtrElem& o) const { return *e.elemHash <  *o.e.elemHash; }
	bool operator<=(const SharedPtrElem& o) const { return *e.elemHash <= *o.e.elemHash; }
	bool operator> (const SharedPtrElem& o) const { return *e.elemHash >  *o.e.elemHash; }
	bool operator>=(const SharedPtrElem& o) const { return *e.elemHash >= *o.e.elemHash; }
	SharedPtrElem(const PtrPair& eIn) : e(eIn) {}
};

class Deduper {
private:
	std::mutex dedup_mutex;
	std::set<FlaggedArraySet*> allArraySets;
	std::thread dedup_thread;
public:
	Deduper()
		: dedup_thread([&]() {
#ifdef PRECISE_BENCH
			return;
#endif
			while (true) {
				bool haveMultipleSets = false;
				{
					std::lock_guard<std::mutex> lock(dedup_mutex);
					haveMultipleSets = allArraySets.size() > 1;
				}

				if (haveMultipleSets) {
					std::list<PtrPair> ptrlist;

					{
						std::lock_guard<std::mutex> lock(dedup_mutex);
						for (FlaggedArraySet* fas : allArraySets) {
							if (fas->allowDups)
								continue;
							if (!fas->mutex.try_lock())
								continue;
							std::lock_guard<WaitCountMutex> lock(fas->mutex, std::adopt_lock);
							for (const auto& e : fas->backingMap) {
								if (fas->mutex.wait_count())
									break;
								assert(e.first.elem);
								ptrlist.push_back(PtrPair(e.first.elem, e.first.elemHash));
							}
						}
					}

					std::set<SharedPtrElem> txset;
					std::map<std::vector<unsigned char>*, PtrPair> duplicateMap;
					std::list<PtrPair> deallocList;
					for (const auto& ptr : ptrlist) {
						assert(ptr.elemHash);
						auto res = txset.insert(SharedPtrElem(ptr));
						if (!res.second && res.first->e.elem != ptr.elem)
							duplicateMap.insert(std::make_pair(&(*ptr.elem), res.first->e));
					}

					int dedups = 0;
					{
						std::lock_guard<std::mutex> lock(dedup_mutex);
						for (FlaggedArraySet* fas : allArraySets) {
							if (fas->allowDups)
								continue;
							if (!fas->mutex.try_lock())
								continue;
							std::lock_guard<WaitCountMutex> lock(fas->mutex, std::adopt_lock);
							for (auto& e : fas->backingMap) {
								if (fas->mutex.wait_count())
									break;
								assert(e.first.elem);
								auto it = duplicateMap.find(&(*e.first.elem));
								if (it != duplicateMap.end()) {
									assert(*it->second.elem == *e.first.elem);
									assert(*it->second.elemHash == *e.first.elemHash);
									deallocList.emplace_back(it->second);
									const_cast<ElemAndFlag&>(e.first).elem.swap(deallocList.back().elem);
									const_cast<ElemAndFlag&>(e.first).elemHash.swap(deallocList.back().elemHash);
									dedups++;
								}
							}
						}
					}
#ifdef FOR_TEST
					if (dedups)
						printf("Deduped %d txn\n", dedups);
#endif
				}
#ifdef FOR_TEST
				std::this_thread::sleep_for(std::chrono::milliseconds(5));
#else
				std::this_thread::sleep_for(std::chrono::milliseconds(5000));
#endif
			}
		})
	{}

	~Deduper() {
		//TODO: close thread
	}

	void addFAS(FlaggedArraySet* fas) {
		std::lock_guard<std::mutex> lock(dedup_mutex);
		allArraySets.insert(fas);
	}

	void removeFAS(FlaggedArraySet* fas) {
		std::lock_guard<std::mutex> lock(dedup_mutex);
		allArraySets.erase(fas);
	}
};

static Deduper* deduper;

FlaggedArraySet::FlaggedArraySet(unsigned int maxSizeIn, bool allowDupsIn) :
		maxSize(maxSizeIn), backingMap(maxSize), allowDups(allowDupsIn) {
	clear();
	if (!deduper)
		deduper = new Deduper();
	deduper->addFAS(this);
}

FlaggedArraySet::~FlaggedArraySet() {
	deduper->removeFAS(this);
	assert(sanity_check());
}

ElemAndFlag::ElemAndFlag(const std::shared_ptr<std::vector<unsigned char> >& elemIn, bool flagIn, bool allowDupsIn, bool setHash) :
	flag(flagIn), allowDups(allowDupsIn), elem(elemIn)
{
	if (setHash) {
		elemHash = std::make_shared<std::vector<unsigned char> >(32);
		double_sha256(&(*elem)[0], &(*elemHash)[0], elem->size());
	}
}
ElemAndFlag::ElemAndFlag(const std::vector<unsigned char>::const_iterator& elemBeginIn, const std::vector<unsigned char>::const_iterator& elemEndIn, bool flagIn, bool allowDupsIn) :
	flag(flagIn), allowDups(allowDupsIn), elemBegin(elemBeginIn), elemEnd(elemEndIn) {}

bool ElemAndFlag::operator == (const ElemAndFlag& o) const {
	if (elem && o.elem) {
		if (allowDups)
			return o.elem == elem;
		bool hashSet = o.elemHash && elemHash;
		return o.elem == elem ||
			(hashSet && *o.elemHash == *elemHash) ||
			(!hashSet && *o.elem == *elem);
	} else {
		std::vector<unsigned char>::const_iterator o_begin, o_end, e_begin, e_end;
		if (elem) {
			e_begin = elem->begin();
			e_end = elem->end();
		} else {
			e_begin = elemBegin;
			e_end = elemEnd;
		}
		if (o.elem) {
			o_begin = o.elem->begin();
			o_end = o.elem->end();
		} else {
			o_begin = o.elemBegin;
			o_end = o.elemEnd;
		}
		return o_end - o_begin == e_end - e_begin && !memcmp(&(*o_begin), &(*e_begin), o_end - o_begin);
	}
}

size_t std::hash<ElemAndFlag>::operator()(const ElemAndFlag& e) const {
	std::vector<unsigned char>::const_iterator it, end;
	if (e.elem) {
		it = e.elem->begin();
		end = e.elem->end();
	} else {
		it = e.elemBegin;
		end = e.elemEnd;
	}

	if (end - it < 5 + 32 + 4) {
		assert(0);
		return 42; // WAT?
	}
	it += 5 + 32 + 4 - 8;
	size_t res = 0;
	static_assert(sizeof(size_t) == 4 || sizeof(size_t) == 8, "Your size_t is neither 32-bit nor 64-bit?");
	for (unsigned int i = 0; i < 8; i += sizeof(size_t)) {
		for (unsigned int j = 0; j < sizeof(size_t); j++)
			res ^= *(it + i + j) << 8*j;
	}
	return res;
}

bool FlaggedArraySet::sanity_check() const {
	size_t size = indexMap.size();
	assert(backingMap.size() == size);
	assert(this->size() == size - to_be_removed.size());

	size_t expected_flag_count = 0;
	for (uint64_t i = 0; i < size; i++) {
		std::unordered_map<ElemAndFlag, uint64_t>::iterator it = indexMap.at(i);
		assert(it != backingMap.end());
		assert(it->second == i + offset);
		assert(backingMap.find(it->first) == it);
		assert(&backingMap.find(it->first)->first == &it->first);
		if (it->first.flag)
			expected_flag_count++;
	}
	assert(expected_flag_count == flag_count);

	ssize_t expected_flags_removed = 0;
	for (size_t i = 0; i < to_be_removed.size(); i++) {
		std::unordered_map<ElemAndFlag, uint64_t>::iterator it = indexMap.at(to_be_removed[i] + i);
		if (it->first.flag)
			expected_flags_removed++;
	}
	assert(expected_flags_removed == flags_to_remove);

	return expected_flags_removed == flags_to_remove && expected_flag_count == flag_count;
}

void FlaggedArraySet::remove_(size_t index) {
	auto& rm = indexMap[index];
	assert(index < indexMap.size());
	if (rm->first.flag)
		flag_count--;

	size_t size = backingMap.size();

	if (index < size/2) {
		for (uint64_t i = 0; i < index; i++)
			indexMap[i]->second++;
		offset++;
	} else
		for (uint64_t i = index + 1; i < size; i++)
			indexMap[i]->second--;
	backingMap.erase(rm);
	indexMap.erase(indexMap.begin() + index);
}

inline void FlaggedArraySet::cleanup_late_remove() const {
	assert(sanity_check());
	if (to_be_removed.size()) {
		for (unsigned int i = 0; i < to_be_removed.size(); i++) {
			assert((unsigned int)to_be_removed[i] < indexMap.size());
			const_cast<FlaggedArraySet*>(this)->remove_(to_be_removed[i]);
		}
		to_be_removed.clear();
		flags_to_remove = 0;
		max_remove = 0;
	}
	assert(sanity_check());
}

bool FlaggedArraySet::contains(const std::shared_ptr<std::vector<unsigned char> >& e) const {
	std::lock_guard<WaitCountMutex> lock(mutex);
	cleanup_late_remove();
	return backingMap.count(ElemAndFlag(e, false, allowDups, false));
}

void FlaggedArraySet::add(const std::shared_ptr<std::vector<unsigned char> >& e, bool flag) {
	ElemAndFlag elem(e, flag, allowDups, true);

	std::lock_guard<WaitCountMutex> lock(mutex);
	cleanup_late_remove();

	auto res = backingMap.insert(std::make_pair(elem, size() + offset));
	if (!res.second)
		return;

	indexMap.push_back(res.first);

	assert(size() <= maxSize + 1);
	while (size() > maxSize)
		remove_(0);

	if (flag)
		flag_count++;

	assert(sanity_check());
}

int FlaggedArraySet::remove(const std::vector<unsigned char>::const_iterator& start, const std::vector<unsigned char>::const_iterator& end) {
	std::lock_guard<WaitCountMutex> lock(mutex);
	cleanup_late_remove();

	auto it = backingMap.find(ElemAndFlag(start, end, false, allowDups));
	if (it == backingMap.end())
		return -1;

	int res = it->second - offset;
	remove_(res);

	assert(sanity_check());
	return res;
}

bool FlaggedArraySet::remove(int index, std::vector<unsigned char>& elemRes, unsigned char* elemHashRes) {
	std::lock_guard<WaitCountMutex> lock(mutex);

	if (index < max_remove)
		cleanup_late_remove();
	int lookup_index = index + to_be_removed.size();

	if ((unsigned int)lookup_index >= indexMap.size())
		return false;

	const ElemAndFlag& e = indexMap[lookup_index]->first;
	assert(e.elem && e.elemHash);
	memcpy(elemHashRes, &(*e.elemHash)[0], 32);
	elemRes = *e.elem;

	if (index >= max_remove) {
		to_be_removed.push_back(index);
		max_remove = index;
		if (e.flag) flags_to_remove++;
	} else {
		cleanup_late_remove();
		remove_(index);
	}

	assert(sanity_check());
	return true;
}

void FlaggedArraySet::clear() {
	std::lock_guard<WaitCountMutex> lock(mutex);
	if (!indexMap.empty() && !backingMap.empty())
		assert(sanity_check());

	flag_count = 0; offset = 0;
	flags_to_remove = 0; max_remove = 0;
	backingMap.clear(); indexMap.clear(); to_be_removed.clear();
}

void FlaggedArraySet::for_all_txn(const std::function<void (const std::shared_ptr<std::vector<unsigned char> >&)> callback) const {
	std::lock_guard<WaitCountMutex> lock(mutex);
	cleanup_late_remove();
	for (const auto& e : indexMap) {
		assert(e->first.elem);
		callback(e->first.elem);
	}
}
<|endoftext|>


<commit_before>#include "a_devdiag.h"
#include "netstructures.h"
#include "tracepacket.h"

void a_devdiag() {
	pcap_if_t *alldevs;
	char errbuf[PCAP_ERRBUF_SIZE];

	if (pcap_findalldevs(&alldevs, errbuf) == -1) {
		printf("Could not find all network devices: %s\n", errbuf);
		exit(1);
	}

	int i = 0;
	pcap_if_t *d;
	for (d = alldevs; d; d = d->next) {
		printf("%d. %s", ++i, d->name);
		if (d->description) {
			printf(" - %s (%s)\n\n", d->description, getDeviceStatus(d));
		}
	}

	int inum;
	printf("Enter the interface number (1-%d): ", i);
	scanf_s("%d", &inum);
	if (inum < 0 || inum > i) {
		printf("Number out of range!\n");
		exit(1);
	}

	for (d = alldevs, i = 0; i< inum - 1; d = d->next, i++);

	pcap_t *adhandle;
	if ((adhandle = pcap_open_live(d->name, 65536, 1, 1000, errbuf)) == NULL) {
		printf("Unable to open the network adapter! %s is not supported by WinPcap\n", d->name);
		pcap_freealldevs(alldevs);
		exit(1);
	}

	if (pcap_datalink(adhandle) != DLT_EN10MB) {
		printf("Not using an ethernet device: output may not always be correct.\n");
	}

	u_int netmask;
	if (d->addresses != NULL) {
		netmask = ((struct sockaddr_in *)(d->addresses->netmask))->sin_addr.S_un.S_addr;
	}
	else {
		netmask = 0xffffff;
	}

	struct bpf_program fcode;
	char a_packetfilter[] = "ip and udp";
	if (pcap_compile(adhandle, &fcode, a_packetfilter, 1, netmask) < 0) {
		printf("Unable to compile packet filter!\n");
		pcap_freealldevs(alldevs);
		exit(1);
	}

	if (pcap_setfilter(adhandle, &fcode) < 0) {
		printf("Unable to set packet filter!\n");
		pcap_freealldevs(alldevs);
		exit(1);
	}

	pcap_freealldevs(alldevs);
	printf("\nStarting active diagnostics...\n\n");

	pcap_loop(adhandle, INFINITE, a_packethandler, NULL);
}

char* getDeviceStatus(pcap_if_t* d) {
	pcap_t *_d;

	if ((_d = pcap_open_live(d->name, 100, 1, 1000, NULL)) == NULL) {
		printf("Unable to open device %s.\n", d->name);
		return "err";
	}

	const u_char *pkt_data;
	struct pcap_pkthdr *header;
	int res;

	int timeoutCount = 0;
	int activeCount = 0;
	for (int i = 0; i < 9; i++) {
		res = pcap_next_ex(_d, &header, &pkt_data);
		if (res == 0) {
			timeoutCount++;
		}
		else {
			activeCount++;
		}
	}
	if (timeoutCount >= activeCount) {
		return "inactive";
	}
	else {
		return "active";
	}
}

int packet_count = 0;

void a_packethandler(u_char *param, const struct pcap_pkthdr *header, const u_char *pkt_data) {
	packet_count++;

	struct tm ltime;
	char timestr[16];
	ip_header *ih;
	udp_header *uh;
	u_int ip_len;
	u_short sport, dport;
	time_t local_tv_sec;

	(VOID)(param);

	local_tv_sec = header->ts.tv_sec;
	localtime_s(&ltime, &local_tv_sec);
	strftime(timestr, sizeof(timestr), "%H:%M:%S", &ltime);

	ih = (ip_header *)(pkt_data + 14);

	ip_len = (ih->ver_ihl & 0xf) * 4;
	uh = (udp_header *)((u_char *)ih + ip_len);

	sport = ntohs(uh->sport);
	dport = ntohs(uh->dport);

	//define current
	currentpacket.count = packet_count;
	currentpacket.timestr = timestr;
	currentpacket.tv_usec = header->ts.tv_usec;
	currentpacket.d_len = header->len;
	currentpacket.t_len = ih->tlen;

	currentpacket.direction = "";

	printf("Packet %i:\n", packet_count);

	printf("\tTimestamp: %s,%.6d\n", timestr, header->ts.tv_usec);
	printf("\tDatagram length: %d\n", header->len);
	printf("\tTotal length: %d\n", ih->tlen);

	printf("\t%d.%d.%d.%d:%d -> %d.%d.%d.%d:%d\n",
		ih->saddr.byte1,
		ih->saddr.byte2,
		ih->saddr.byte3,
		ih->saddr.byte4,
		sport,
		ih->daddr.byte1,
		ih->daddr.byte2,
		ih->daddr.byte3,
		ih->daddr.byte4,
		dport);
}<commit_msg>Finished putting all data into tracepacket struct<commit_after>#include <string>
#include <sstream>
#include "a_devdiag.h"
#include "netstructures.h"
#include "tracepacket.h"

void a_devdiag() {
	pcap_if_t *alldevs;
	char errbuf[PCAP_ERRBUF_SIZE];

	if (pcap_findalldevs(&alldevs, errbuf) == -1) {
		printf("Could not find all network devices: %s\n", errbuf);
		exit(1);
	}

	int i = 0;
	pcap_if_t *d;
	for (d = alldevs; d; d = d->next) {
		printf("%d. %s", ++i, d->name);
		if (d->description) {
			printf(" - %s (%s)\n\n", d->description, getDeviceStatus(d));
		}
	}

	int inum;
	printf("Enter the interface number (1-%d): ", i);
	scanf_s("%d", &inum);
	if (inum < 0 || inum > i) {
		printf("Number out of range!\n");
		exit(1);
	}

	for (d = alldevs, i = 0; i< inum - 1; d = d->next, i++);

	pcap_t *adhandle;
	if ((adhandle = pcap_open_live(d->name, 65536, 1, 1000, errbuf)) == NULL) {
		printf("Unable to open the network adapter! %s is not supported by WinPcap\n", d->name);
		pcap_freealldevs(alldevs);
		exit(1);
	}

	if (pcap_datalink(adhandle) != DLT_EN10MB) {
		printf("Not using an ethernet device: output may not always be correct.\n");
	}

	u_int netmask;
	if (d->addresses != NULL) {
		netmask = ((struct sockaddr_in *)(d->addresses->netmask))->sin_addr.S_un.S_addr;
	}
	else {
		netmask = 0xffffff;
	}

	struct bpf_program fcode;
	char a_packetfilter[] = "ip and udp";
	if (pcap_compile(adhandle, &fcode, a_packetfilter, 1, netmask) < 0) {
		printf("Unable to compile packet filter!\n");
		pcap_freealldevs(alldevs);
		exit(1);
	}

	if (pcap_setfilter(adhandle, &fcode) < 0) {
		printf("Unable to set packet filter!\n");
		pcap_freealldevs(alldevs);
		exit(1);
	}

	pcap_freealldevs(alldevs);
	printf("\nStarting active diagnostics...\n\n");

	pcap_loop(adhandle, INFINITE, a_packethandler, NULL);
}

char* getDeviceStatus(pcap_if_t* d) {
	pcap_t *_d;

	if ((_d = pcap_open_live(d->name, 100, 1, 1000, NULL)) == NULL) {
		printf("Unable to open device %s.\n", d->name);
		return "err";
	}

	const u_char *pkt_data;
	struct pcap_pkthdr *header;
	int res;

	int timeoutCount = 0;
	int activeCount = 0;
	for (int i = 0; i < 9; i++) {
		res = pcap_next_ex(_d, &header, &pkt_data);
		if (res == 0) {
			timeoutCount++;
		}
		else {
			activeCount++;
		}
	}
	if (timeoutCount >= activeCount) {
		return "inactive";
	}
	else {
		return "active";
	}
}

int packet_count = 0;

void a_packethandler(u_char *param, const struct pcap_pkthdr *header, const u_char *pkt_data) {
	packet_count++;

	struct tm ltime;
	char timestr[16];
	ip_header *ih;
	udp_header *uh;
	u_int ip_len;
	u_short sport, dport;
	time_t local_tv_sec;

	(VOID)(param);

	local_tv_sec = header->ts.tv_sec;
	localtime_s(&ltime, &local_tv_sec);
	strftime(timestr, sizeof(timestr), "%H:%M:%S", &ltime);

	ih = (ip_header *)(pkt_data + 14);

	ip_len = (ih->ver_ihl & 0xf) * 4;
	uh = (udp_header *)((u_char *)ih + ip_len);

	sport = ntohs(uh->sport);
	dport = ntohs(uh->dport);

	//define current
	currentpacket.count = packet_count;
	currentpacket.timestr = timestr;
	currentpacket.tv_usec = header->ts.tv_usec;
	currentpacket.d_len = header->len;
	currentpacket.t_len = ih->tlen;
	std::ostringstream oss;
	oss << "\t";
	oss << ih->saddr.byte1;
	oss << ".";
	oss << ih->saddr.byte2;
	oss << ".";
	oss << ih->saddr.byte3;
	oss << ".";
	oss << ih->saddr.byte4;
	oss << ":";
	oss << sport;
	oss << " -> ";
	oss << ih->daddr.byte1;
	oss << ".";
	oss << ih->daddr.byte2;
	oss << ".";
	oss << ih->daddr.byte3;
	oss << ".";
	oss << ih->daddr.byte4;
	oss << ":";
	oss << dport;
	oss << "\n";
	currentpacket.direction = (char*)oss.str().c_str();

	printf("Packet %i:\n", packet_count);

	printf("\tTimestamp: %s,%.6d\n", timestr, header->ts.tv_usec);
	printf("\tDatagram length: %d\n", header->len);
	printf("\tTotal length: %d\n", ih->tlen);

	printf("\t%d.%d.%d.%d:%d -> %d.%d.%d.%d:%d\n",
		ih->saddr.byte1,
		ih->saddr.byte2,
		ih->saddr.byte3,
		ih->saddr.byte4,
		sport,
		ih->daddr.byte1,
		ih->daddr.byte2,
		ih->daddr.byte3,
		ih->daddr.byte4,
		dport);
}<|endoftext|>


<commit_before>#pragma once

namespace clay {

template<class T>
class Pointer {
    T *p;
public :
    Pointer()
        : p(0) {}
    Pointer(T *p)
        : p(p) {
        if (p)
            p->incRef();
    }
    Pointer(const Pointer<T> &other)
        : p(other.p) {
        if (p)
            p->incRef();
    }
    ~Pointer() {
        if (p)
            p->decRef();
    }
    Pointer<T> &operator=(const Pointer<T> &other) {
        T *q = other.p;
        if (q) q->incRef();
        if (p) p->decRef();
        p = q;
        return *this;
    }
    T &operator*() { return *p; }
    const T &operator*() const { return *p; }
    T *operator->() { return p; }
    const T *operator->() const { return p; }
    T *ptr() const { return p; }
    bool operator!() const { return p == 0; }
    bool operator==(const Pointer<T> &other) const {
        return p == other.p;
    }
    bool operator!=(const Pointer<T> &other) const {
        return p != other.p;
    }
    bool operator<(const Pointer<T> &other) const {
        return p < other.p;
    }
};

struct RefCounted {
private:
    int refCount;

public:
    RefCounted()
        : refCount(0) {}
    RefCounted(const RefCounted& that): refCount(0) {}
    void incRef() { ++refCount; }
    void decRef() {
        if (--refCount == 0) {
            delete this;
        }
    }
    int getRefCount() { return refCount; }

    virtual ~RefCounted() {}
};

}
<commit_msg>Change Pointer const operator semantics<commit_after>#pragma once

namespace clay {

template<class T>
class Pointer {
    T *p;
public :
    Pointer()
        : p(0) {}
    Pointer(T *p)
        : p(p) {
        if (p)
            p->incRef();
    }
    Pointer(const Pointer<T> &other)
        : p(other.p) {
        if (p)
            p->incRef();
    }
    ~Pointer() {
        if (p)
            p->decRef();
    }
    Pointer<T> &operator=(const Pointer<T> &other) {
        T *q = other.p;
        if (q) q->incRef();
        if (p) p->decRef();
        p = q;
        return *this;
    }
    T &operator*() const { return *p; }
    T *operator->() const { return p; }
    T *ptr() const { return p; }
    bool operator!() const { return p == 0; }
    bool operator==(const Pointer<T> &other) const {
        return p == other.p;
    }
    bool operator!=(const Pointer<T> &other) const {
        return p != other.p;
    }
    bool operator<(const Pointer<T> &other) const {
        return p < other.p;
    }
};

struct RefCounted {
private:
    int refCount;

public:
    RefCounted()
        : refCount(0) {}
    RefCounted(const RefCounted& that): refCount(0) {}
    void incRef() { ++refCount; }
    void decRef() {
        if (--refCount == 0) {
            delete this;
        }
    }
    int getRefCount() { return refCount; }

    virtual ~RefCounted() {}
};

}
<|endoftext|>


<commit_before>/**
 *  ofxCsv
 *  Inspired and based on Ben Fry's [table class](http://benfry.com/writing/map/Table.pde)
 *
 *
 *  The MIT License
 *
 *  Copyright (c) 2011-2012 Paul Vollmer, http://www.wng.cc
 *
 *  Permission is hereby granted, free of charge, to any person obtaining a copy
 *  of this software and associated documentation files (the "Software"), to deal
 *  in the Software without restriction, including without limitation the rights
 *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 *  copies of the Software, and to permit persons to whom the Software is
 *  furnished to do so, subject to the following conditions:
 *
 *  The above copyright notice and this permission notice shall be included in
 *  all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 *  THE SOFTWARE.
 *
 *
 *  @testet_oF          0071
 *  @testet_plattform   MacOs 10.6+
 *                      ??? Win
 *                      ??? Linux
 *  @dependencies
 *  @modified           2012.06.28
 *  @version            0.1.3
 */

#include "ofMain.h"
#include "testApp.h"
#include "ofAppGlutWindow.h"

int main( ){

    ofAppGlutWindow window;
	ofSetupOpenGL(&window, 1024,768, OF_WINDOW);			// <-------- setup the GL context

	// this kicks off the running of my app
	// can be OF_WINDOW or OF_FULLSCREEN
	// pass in width and height too:
	ofRunApp( new testApp());

}
<commit_msg>Update main.cpp<commit_after>/**
 *  ofxCsv
 *  Inspired and based on Ben Fry's [table class](http://benfry.com/writing/map/Table.pde)
 *
 *
 *  The MIT License
 *
 *  Copyright (c) 2011-2014 Paul Vollmer, http://www.wng.cc
 *
 *  Permission is hereby granted, free of charge, to any person obtaining a copy
 *  of this software and associated documentation files (the "Software"), to deal
 *  in the Software without restriction, including without limitation the rights
 *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 *  copies of the Software, and to permit persons to whom the Software is
 *  furnished to do so, subject to the following conditions:
 *
 *  The above copyright notice and this permission notice shall be included in
 *  all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 *  THE SOFTWARE.
 *
 *
 *  @testet_oF          0071
 *  @testet_plattform   MacOs 10.6+
 *                      ??? Win
 *                      ??? Linux
 *  @dependencies
 *  @modified           2012.06.28
 *  @version            0.1.3
 */

#include "ofMain.h"
#include "testApp.h"
#include "ofAppGlutWindow.h"

int main( ){

    ofAppGlutWindow window;
	ofSetupOpenGL(&window, 1024,768, OF_WINDOW);			// <-------- setup the GL context

	// this kicks off the running of my app
	// can be OF_WINDOW or OF_FULLSCREEN
	// pass in width and height too:
	ofRunApp( new testApp());

}
<|endoftext|>


<commit_before>// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html

#include "precomp.hpp"

#include <stdio.h>
#include <string.h>
#include <fstream>
#include <iostream>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <android/log.h>

#include "media/NdkMediaCodec.h"
#include "media/NdkMediaExtractor.h"

#define INPUT_TIMEOUT_MS 2000

#define COLOR_FormatYUV420Planar 19
#define COLOR_FormatYUV420SemiPlanar 21

using namespace cv;

#define TAG "NativeCodec"
#define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)

static inline void deleter_AMediaExtractor(AMediaExtractor *extractor) {
    AMediaExtractor_delete(extractor);
}

static inline void deleter_AMediaCodec(AMediaCodec *codec) {
    AMediaCodec_stop(codec);
    AMediaCodec_delete(codec);
}

static inline void deleter_AMediaFormat(AMediaFormat *format) {
    AMediaFormat_delete(format);
}

class AndroidMediaNdkCapture : public IVideoCapture
{

public:
    AndroidMediaNdkCapture():
        sawInputEOS(false), sawOutputEOS(false),
        frameWidth(0), frameHeight(0), colorFormat(0) {}
    std::shared_ptr<AMediaExtractor> mediaExtractor;
    std::shared_ptr<AMediaCodec> mediaCodec;
    bool sawInputEOS;
    bool sawOutputEOS;
    int32_t frameWidth;
    int32_t frameHeight;
    int32_t colorFormat;
    std::vector<uint8_t> buffer;

    ~AndroidMediaNdkCapture() { cleanUp(); }

    bool decodeFrame() {
        while (!sawInputEOS || !sawOutputEOS) {
            if (!sawInputEOS) {
                auto bufferIndex = AMediaCodec_dequeueInputBuffer(mediaCodec.get(), INPUT_TIMEOUT_MS);
                LOGV("input buffer %zd", bufferIndex);
                if (bufferIndex >= 0) {
                    size_t bufferSize;
                    auto inputBuffer = AMediaCodec_getInputBuffer(mediaCodec.get(), bufferIndex, &bufferSize);
                    auto sampleSize = AMediaExtractor_readSampleData(mediaExtractor.get(), inputBuffer, bufferSize);
                    if (sampleSize < 0) {
                        sampleSize = 0;
                        sawInputEOS = true;
                        LOGV("EOS");
                    }
                    auto presentationTimeUs = AMediaExtractor_getSampleTime(mediaExtractor.get());

                    AMediaCodec_queueInputBuffer(mediaCodec.get(), bufferIndex, 0, sampleSize,
                        presentationTimeUs, sawInputEOS ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0);
                    AMediaExtractor_advance(mediaExtractor.get());
                }
            }

            if (!sawOutputEOS) {
                AMediaCodecBufferInfo info;
                auto bufferIndex = AMediaCodec_dequeueOutputBuffer(mediaCodec.get(), &info, 0);
                if (bufferIndex >= 0) {
                    size_t bufferSize = 0;
                    auto mediaFormat = std::shared_ptr<AMediaFormat>(AMediaCodec_getOutputFormat(mediaCodec.get()), deleter_AMediaFormat);
                    AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_WIDTH, &frameWidth);
                    AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_HEIGHT, &frameHeight);
                    AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat);
                    uint8_t* codecBuffer = AMediaCodec_getOutputBuffer(mediaCodec.get(), bufferIndex, &bufferSize);
                    buffer = std::vector<uint8_t>(codecBuffer + info.offset, codecBuffer + bufferSize);
                    LOGV("colorFormat: %d", colorFormat);
                    LOGV("buffer size: %zu", bufferSize);
                    LOGV("width (frame): %d", frameWidth);
                    LOGV("height (frame): %d", frameHeight);
                    if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
                        LOGV("output EOS");
                        sawOutputEOS = true;
                    }
                    AMediaCodec_releaseOutputBuffer(mediaCodec.get(), bufferIndex, info.size != 0);
                    return true;
                } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
                    LOGV("output buffers changed");
                } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
                    auto format = AMediaCodec_getOutputFormat(mediaCodec.get());
                    LOGV("format changed to: %s", AMediaFormat_toString(format));
                    AMediaFormat_delete(format);
                } else if (bufferIndex == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
                    LOGV("no output buffer right now");
                } else {
                    LOGV("unexpected info code: %zd", bufferIndex);
                }
            }
        }
        return false;
    }

    bool isOpened() const CV_OVERRIDE { return mediaCodec.get() != nullptr; }

    int getCaptureDomain() CV_OVERRIDE { return CAP_ANDROID; }

    bool grabFrame() CV_OVERRIDE
    {
        // clear the previous frame
        buffer.clear();
        return decodeFrame();
    }

    bool retrieveFrame(int, OutputArray out) CV_OVERRIDE
    {
        if (buffer.empty()) {
            return false;
        }
        Mat yuv(frameHeight + frameHeight/2, frameWidth, CV_8UC1, buffer.data());
        if (colorFormat == COLOR_FormatYUV420Planar) {
            cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_YV12);
        } else if (colorFormat == COLOR_FormatYUV420SemiPlanar) {
            cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_NV21);
        } else {
            LOGE("Unsupported video format: %d", colorFormat);
            return false;
        }
        return true;
    }

    double getProperty(int property_id) const CV_OVERRIDE
    {
        switch (property_id)
        {
            case CV_CAP_PROP_FRAME_WIDTH: return frameWidth;
            case CV_CAP_PROP_FRAME_HEIGHT: return frameHeight;
        }
        return 0;
    }

    bool setProperty(int /* property_id */, double /* value */) CV_OVERRIDE
    {
        return false;
    }

    bool initCapture(const char * filename)
    {
        struct stat statBuffer;
        if (stat(filename, &statBuffer) != 0) {
            LOGE("failed to stat file: %s (%s)", filename, strerror(errno));
            return false;
        }

        int fd = open(filename, O_RDONLY);

        if (fd < 0) {
            LOGE("failed to open file: %s %d (%s)", filename, fd, strerror(errno));
            return false;
        }

        mediaExtractor = std::shared_ptr<AMediaExtractor>(AMediaExtractor_new(), deleter_AMediaExtractor);
        if (!mediaExtractor) {
            return false;
        }
        media_status_t err = AMediaExtractor_setDataSourceFd(mediaExtractor.get(), fd, 0, statBuffer.st_size);
        close(fd);
        if (err != AMEDIA_OK) {
            LOGV("setDataSource error: %d", err);
            return false;
        }

        int numtracks = AMediaExtractor_getTrackCount(mediaExtractor.get());

        LOGV("input has %d tracks", numtracks);
        for (int i = 0; i < numtracks; i++) {
            auto format = std::shared_ptr<AMediaFormat>(AMediaExtractor_getTrackFormat(mediaExtractor.get(), i), deleter_AMediaFormat);
            if (!format) {
                continue;
            }
            const char *s = AMediaFormat_toString(format.get());
            LOGV("track %d format: %s", i, s);
            const char *mime;
            if (!AMediaFormat_getString(format.get(), AMEDIAFORMAT_KEY_MIME, &mime)) {
                LOGV("no mime type");
            } else if (!strncmp(mime, "video/", 6)) {
                int32_t trackWidth, trackHeight;
                AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_WIDTH, &trackWidth);
                AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_HEIGHT, &trackHeight);
                LOGV("width (track): %d", trackWidth);
                LOGV("height (track): %d", trackHeight);
                if (AMediaExtractor_selectTrack(mediaExtractor.get(), i) != AMEDIA_OK) {
                    continue;
                }
                mediaCodec = std::shared_ptr<AMediaCodec>(AMediaCodec_createDecoderByType(mime), deleter_AMediaCodec);
                if (!mediaCodec) {
                    continue;
                }
                if (AMediaCodec_configure(mediaCodec.get(), format.get(), NULL, NULL, 0) != AMEDIA_OK) {
                    continue;
                }
                sawInputEOS = false;
                sawOutputEOS = false;
                if (AMediaCodec_start(mediaCodec.get()) != AMEDIA_OK) {
                    continue;
                }
                return true;
            }
        }

        return false;
    }

    void cleanUp() {
        sawInputEOS = true;
        sawOutputEOS = true;
        frameWidth = 0;
        frameHeight = 0;
        colorFormat = 0;
    }
};

/****************** Implementation of interface functions ********************/

Ptr<IVideoCapture> cv::createAndroidCapture_file(const std::string &filename) {
    Ptr<AndroidMediaNdkCapture> res = makePtr<AndroidMediaNdkCapture>();
    if (res && res->initCapture(filename.c_str()))
        return res;
    return Ptr<IVideoCapture>();
}
<commit_msg>videoio: drop unnecessary offset for accessing video output buffer<commit_after>// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html

#include "precomp.hpp"

#include <stdio.h>
#include <string.h>
#include <fstream>
#include <iostream>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <android/log.h>

#include "media/NdkMediaCodec.h"
#include "media/NdkMediaExtractor.h"

#define INPUT_TIMEOUT_MS 2000

#define COLOR_FormatYUV420Planar 19
#define COLOR_FormatYUV420SemiPlanar 21

using namespace cv;

#define TAG "NativeCodec"
#define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)

static inline void deleter_AMediaExtractor(AMediaExtractor *extractor) {
    AMediaExtractor_delete(extractor);
}

static inline void deleter_AMediaCodec(AMediaCodec *codec) {
    AMediaCodec_stop(codec);
    AMediaCodec_delete(codec);
}

static inline void deleter_AMediaFormat(AMediaFormat *format) {
    AMediaFormat_delete(format);
}

class AndroidMediaNdkCapture : public IVideoCapture
{

public:
    AndroidMediaNdkCapture():
        sawInputEOS(false), sawOutputEOS(false),
        frameWidth(0), frameHeight(0), colorFormat(0) {}
    std::shared_ptr<AMediaExtractor> mediaExtractor;
    std::shared_ptr<AMediaCodec> mediaCodec;
    bool sawInputEOS;
    bool sawOutputEOS;
    int32_t frameWidth;
    int32_t frameHeight;
    int32_t colorFormat;
    std::vector<uint8_t> buffer;

    ~AndroidMediaNdkCapture() { cleanUp(); }

    bool decodeFrame() {
        while (!sawInputEOS || !sawOutputEOS) {
            if (!sawInputEOS) {
                auto bufferIndex = AMediaCodec_dequeueInputBuffer(mediaCodec.get(), INPUT_TIMEOUT_MS);
                LOGV("input buffer %zd", bufferIndex);
                if (bufferIndex >= 0) {
                    size_t bufferSize;
                    auto inputBuffer = AMediaCodec_getInputBuffer(mediaCodec.get(), bufferIndex, &bufferSize);
                    auto sampleSize = AMediaExtractor_readSampleData(mediaExtractor.get(), inputBuffer, bufferSize);
                    if (sampleSize < 0) {
                        sampleSize = 0;
                        sawInputEOS = true;
                        LOGV("EOS");
                    }
                    auto presentationTimeUs = AMediaExtractor_getSampleTime(mediaExtractor.get());

                    AMediaCodec_queueInputBuffer(mediaCodec.get(), bufferIndex, 0, sampleSize,
                        presentationTimeUs, sawInputEOS ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0);
                    AMediaExtractor_advance(mediaExtractor.get());
                }
            }

            if (!sawOutputEOS) {
                AMediaCodecBufferInfo info;
                auto bufferIndex = AMediaCodec_dequeueOutputBuffer(mediaCodec.get(), &info, 0);
                if (bufferIndex >= 0) {
                    size_t bufferSize = 0;
                    auto mediaFormat = std::shared_ptr<AMediaFormat>(AMediaCodec_getOutputFormat(mediaCodec.get()), deleter_AMediaFormat);
                    AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_WIDTH, &frameWidth);
                    AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_HEIGHT, &frameHeight);
                    AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat);
                    uint8_t* codecBuffer = AMediaCodec_getOutputBuffer(mediaCodec.get(), bufferIndex, &bufferSize);
                    buffer = std::vector<uint8_t>(codecBuffer, codecBuffer + bufferSize);
                    LOGV("colorFormat: %d", colorFormat);
                    LOGV("buffer size: %zu", bufferSize);
                    LOGV("width (frame): %d", frameWidth);
                    LOGV("height (frame): %d", frameHeight);
                    if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
                        LOGV("output EOS");
                        sawOutputEOS = true;
                    }
                    AMediaCodec_releaseOutputBuffer(mediaCodec.get(), bufferIndex, info.size != 0);
                    return true;
                } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
                    LOGV("output buffers changed");
                } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
                    auto format = AMediaCodec_getOutputFormat(mediaCodec.get());
                    LOGV("format changed to: %s", AMediaFormat_toString(format));
                    AMediaFormat_delete(format);
                } else if (bufferIndex == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
                    LOGV("no output buffer right now");
                } else {
                    LOGV("unexpected info code: %zd", bufferIndex);
                }
            }
        }
        return false;
    }

    bool isOpened() const CV_OVERRIDE { return mediaCodec.get() != nullptr; }

    int getCaptureDomain() CV_OVERRIDE { return CAP_ANDROID; }

    bool grabFrame() CV_OVERRIDE
    {
        // clear the previous frame
        buffer.clear();
        return decodeFrame();
    }

    bool retrieveFrame(int, OutputArray out) CV_OVERRIDE
    {
        if (buffer.empty()) {
            return false;
        }
        Mat yuv(frameHeight + frameHeight/2, frameWidth, CV_8UC1, buffer.data());
        if (colorFormat == COLOR_FormatYUV420Planar) {
            cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_YV12);
        } else if (colorFormat == COLOR_FormatYUV420SemiPlanar) {
            cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_NV21);
        } else {
            LOGE("Unsupported video format: %d", colorFormat);
            return false;
        }
        return true;
    }

    double getProperty(int property_id) const CV_OVERRIDE
    {
        switch (property_id)
        {
            case CV_CAP_PROP_FRAME_WIDTH: return frameWidth;
            case CV_CAP_PROP_FRAME_HEIGHT: return frameHeight;
        }
        return 0;
    }

    bool setProperty(int /* property_id */, double /* value */) CV_OVERRIDE
    {
        return false;
    }

    bool initCapture(const char * filename)
    {
        struct stat statBuffer;
        if (stat(filename, &statBuffer) != 0) {
            LOGE("failed to stat file: %s (%s)", filename, strerror(errno));
            return false;
        }

        int fd = open(filename, O_RDONLY);

        if (fd < 0) {
            LOGE("failed to open file: %s %d (%s)", filename, fd, strerror(errno));
            return false;
        }

        mediaExtractor = std::shared_ptr<AMediaExtractor>(AMediaExtractor_new(), deleter_AMediaExtractor);
        if (!mediaExtractor) {
            return false;
        }
        media_status_t err = AMediaExtractor_setDataSourceFd(mediaExtractor.get(), fd, 0, statBuffer.st_size);
        close(fd);
        if (err != AMEDIA_OK) {
            LOGV("setDataSource error: %d", err);
            return false;
        }

        int numtracks = AMediaExtractor_getTrackCount(mediaExtractor.get());

        LOGV("input has %d tracks", numtracks);
        for (int i = 0; i < numtracks; i++) {
            auto format = std::shared_ptr<AMediaFormat>(AMediaExtractor_getTrackFormat(mediaExtractor.get(), i), deleter_AMediaFormat);
            if (!format) {
                continue;
            }
            const char *s = AMediaFormat_toString(format.get());
            LOGV("track %d format: %s", i, s);
            const char *mime;
            if (!AMediaFormat_getString(format.get(), AMEDIAFORMAT_KEY_MIME, &mime)) {
                LOGV("no mime type");
            } else if (!strncmp(mime, "video/", 6)) {
                int32_t trackWidth, trackHeight;
                AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_WIDTH, &trackWidth);
                AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_HEIGHT, &trackHeight);
                LOGV("width (track): %d", trackWidth);
                LOGV("height (track): %d", trackHeight);
                if (AMediaExtractor_selectTrack(mediaExtractor.get(), i) != AMEDIA_OK) {
                    continue;
                }
                mediaCodec = std::shared_ptr<AMediaCodec>(AMediaCodec_createDecoderByType(mime), deleter_AMediaCodec);
                if (!mediaCodec) {
                    continue;
                }
                if (AMediaCodec_configure(mediaCodec.get(), format.get(), NULL, NULL, 0) != AMEDIA_OK) {
                    continue;
                }
                sawInputEOS = false;
                sawOutputEOS = false;
                if (AMediaCodec_start(mediaCodec.get()) != AMEDIA_OK) {
                    continue;
                }
                return true;
            }
        }

        return false;
    }

    void cleanUp() {
        sawInputEOS = true;
        sawOutputEOS = true;
        frameWidth = 0;
        frameHeight = 0;
        colorFormat = 0;
    }
};

/****************** Implementation of interface functions ********************/

Ptr<IVideoCapture> cv::createAndroidCapture_file(const std::string &filename) {
    Ptr<AndroidMediaNdkCapture> res = makePtr<AndroidMediaNdkCapture>();
    if (res && res->initCapture(filename.c_str()))
        return res;
    return Ptr<IVideoCapture>();
}
<|endoftext|>