<commit_before>#ifndef GAMELOGIC_FOUR_IN_A_LINE_ASYNCHRONOUSARTIFICIALINTELLIGENCE_HPP
#define GAMELOGIC_FOUR_IN_A_LINE_ASYNCHRONOUSARTIFICIALINTELLIGENCE_HPP

#include "ArtificialIntelligence.hpp"

#include <atomic>
#include <future>
#include <chrono>

namespace GameLogic
{
namespace FourInALine
{

/**
 * Artificial intelligence which does the computing in a separate thread.
 *
 * This class extends the artificial intelligence and modifies it so that the computation of the
 * next move is done in a separate thread.
 *
 * The next move can be computed with computeNextMoveAsynchronously(). Use isNextMoveRead() to check
 * whether the result is ready and valid. The resulting move can then be retrieved with
 * getNextMove(), which also accepts an optional timeout parameter to block until the timeout
 * duration has elapsed or the result is ready. Default is a timeout of 0, which means no blocking.
 * When the timeout duration has elapsed and the result is not yet ready, it returns an
 * INVALID_RESULT.
 *
 * It is possible to check whether the AI is currently computing with isComputing() and the
 * computation can be cancelled with cancelComputation(). When the computation is cancelled, the
 * result will be INVALID_RESULT. Use isNextMoveValid() to check whether the next move is valid or
 * INVALID_RESULT.
 *
 * This class is reentrant but not thread-safe.
 */
class AsynchronousArtificialIntelligence : public ArtificialIntelligence
{
	public:
		AsynchronousArtificialIntelligence(unsigned int searchDepth);
		virtual ~AsynchronousArtificialIntelligence();

		void computeNextMoveAsynchronously(const Game& game);
		bool isNextMoveReady();
		bool isNextMoveValid();
		bool isComputing() const;
		unsigned int getNextMove(std::chrono::milliseconds timeout = std::chrono::milliseconds(0));

		void cancelComputation();

		/**
		 * Invalid result returned when the next move is not yet ready or the computation was
		 * cancelled.
		 */
		static const unsigned int INVALID_RESULT;

	private:
		virtual unsigned int computeNextMove(const Game& game) const override;
		virtual ScoredMovesType minimax(Node node, const Board& board, PlayerType player,
		                                unsigned int depth, ScoreType alpha, ScoreType beta) const override;

		/**
		 * Whether the computation should be cancelled or not.
		 */
		std::atomic<bool> cancel;

		/**
		 * Whether a computation is running or not.
		 *
		 * Mutable so that the minimax() method can stay const.
		 */
		mutable std::atomic<bool> running;

		/**
		 * Future used to compute the next task.
		 */
		std::future<unsigned int> nextMoveTask;

		/**
		 * The result of the last computation.
		 *
		 * Either a valid move or INVALID_RESULT.
		 */
		unsigned int lastResult;
};

}
}

#endif // GAMELOGIC_FOUR_IN_A_LINE_ASYNCHRONOUSARTIFICIALINTELLIGENCE_HPP
<commit_msg>Fixed typo.<commit_after>#ifndef GAMELOGIC_FOUR_IN_A_LINE_ASYNCHRONOUSARTIFICIALINTELLIGENCE_HPP
#define GAMELOGIC_FOUR_IN_A_LINE_ASYNCHRONOUSARTIFICIALINTELLIGENCE_HPP

#include "ArtificialIntelligence.hpp"

#include <atomic>
#include <future>
#include <chrono>

namespace GameLogic
{
namespace FourInALine
{

/**
 * Artificial intelligence which does the computing in a separate thread.
 *
 * This class extends the artificial intelligence and modifies it so that the computation of the
 * next move is done in a separate thread.
 *
 * The next move can be computed with computeNextMoveAsynchronously(). Use isNextMoveReady() to check
 * whether the result is ready and valid. The resulting move can then be retrieved with
 * getNextMove(), which also accepts an optional timeout parameter to block until the timeout
 * duration has elapsed or the result is ready. Default is a timeout of 0, which means no blocking.
 * When the timeout duration has elapsed and the result is not yet ready, it returns an
 * INVALID_RESULT.
 *
 * It is possible to check whether the AI is currently computing with isComputing() and the
 * computation can be cancelled with cancelComputation(). When the computation is cancelled, the
 * result will be INVALID_RESULT. Use isNextMoveValid() to check whether the next move is valid or
 * INVALID_RESULT.
 *
 * This class is reentrant but not thread-safe.
 */
class AsynchronousArtificialIntelligence : public ArtificialIntelligence
{
	public:
		AsynchronousArtificialIntelligence(unsigned int searchDepth);
		virtual ~AsynchronousArtificialIntelligence();

		void computeNextMoveAsynchronously(const Game& game);
		bool isNextMoveReady();
		bool isNextMoveValid();
		bool isComputing() const;
		unsigned int getNextMove(std::chrono::milliseconds timeout = std::chrono::milliseconds(0));

		void cancelComputation();

		/**
		 * Invalid result returned when the next move is not yet ready or the computation was
		 * cancelled.
		 */
		static const unsigned int INVALID_RESULT;

	private:
		virtual unsigned int computeNextMove(const Game& game) const override;
		virtual ScoredMovesType minimax(Node node, const Board& board, PlayerType player,
		                                unsigned int depth, ScoreType alpha, ScoreType beta) const override;

		/**
		 * Whether the computation should be cancelled or not.
		 */
		std::atomic<bool> cancel;

		/**
		 * Whether a computation is running or not.
		 *
		 * Mutable so that the minimax() method can stay const.
		 */
		mutable std::atomic<bool> running;

		/**
		 * Future used to compute the next task.
		 */
		std::future<unsigned int> nextMoveTask;

		/**
		 * The result of the last computation.
		 *
		 * Either a valid move or INVALID_RESULT.
		 */
		unsigned int lastResult;
};

}
}

#endif // GAMELOGIC_FOUR_IN_A_LINE_ASYNCHRONOUSARTIFICIALINTELLIGENCE_HPP
<|endoftext|>


<commit_before>#include <openssl/evp.h>

#include <common/factory.h>

#include <crypto/crypto_encryption.h>

namespace {
	class SessionEVP : public CryptoEncryption::Session {
		LogHandle log_;
		const EVP_CIPHER *cipher_;
		EVP_CIPHER_CTX ctx_;
	public:
		SessionEVP(const EVP_CIPHER *xcipher)
		: log_("/crypto/encryption/session/openssl"),
		  cipher_(xcipher),
		  ctx_()
		{
			EVP_CIPHER_CTX_init(&ctx_);
		}

		~SessionEVP()
		{
			EVP_CIPHER_CTX_cleanup(&ctx_);
		}

		unsigned block_size(void) const
		{
			return (EVP_CIPHER_block_size(cipher_));
		}

		unsigned key_size(void) const
		{
			return (EVP_CIPHER_key_length(cipher_));
		}

		unsigned iv_size(void) const
		{
			return (EVP_CIPHER_iv_length(cipher_));
		}

		Session *clone(void) const
		{
			return (new SessionEVP(cipher_));
		}

		bool initialize(CryptoEncryption::Operation operation, const Buffer *key, const Buffer *iv)
		{
			if (key->length() < (size_t)EVP_CIPHER_key_length(cipher_))
				return (false);

			if (iv->length() < (size_t)EVP_CIPHER_iv_length(cipher_))
				return (false);

			int enc;
			switch (operation) {
			case CryptoEncryption::Encrypt:
				enc = 1;
				break;
			case CryptoEncryption::Decrypt:
				enc = 0;
				break;
			default:
				return (false);
			}

			uint8_t keydata[key->length()];
			key->copyout(keydata, sizeof keydata);

			uint8_t ivdata[iv->length()];
			iv->copyout(ivdata, sizeof ivdata);

			int rv = EVP_CipherInit(&ctx_, cipher_, keydata, ivdata, enc);
			if (rv == 0)
				return (false);

			return (true);
		}

		bool cipher(Buffer *out, const Buffer *in)
		{
			/*
			 * We process a single, large, linear byte buffer here rather
			 * than going a BufferSegment at a time, even though the byte
			 * buffer is less efficient than some alternatives, because
			 * there are padding and buffering implications if each
			 * BufferSegment's length is not modular to the block size.
			 */
			uint8_t indata[in->length()];
			in->copyout(indata, sizeof indata);

			uint8_t outdata[sizeof indata];
			int rv = EVP_Cipher(&ctx_, outdata, indata, sizeof indata);
			if (rv == 0)
				return (false);
			out->append(outdata, sizeof outdata);
			return (true);
		}

		Action *submit(Buffer *in, EventCallback *cb)
		{
			Buffer out;
			if (!cipher(&out, in)) {
				in->clear();
				cb->param(Event::Error);
				return (cb->schedule());
			}
			in->clear();
			cb->param(Event(Event::Done, out));
			return (cb->schedule());
		}
	};

	class MethodOpenSSL : public CryptoEncryption::Method {
		LogHandle log_;
		FactoryMap<CryptoEncryption::Cipher, CryptoEncryption::Session> cipher_map_;
	public:
		MethodOpenSSL(void)
		: CryptoEncryption::Method("OpenSSL"),
		  log_("/crypto/encryption/openssl"),
		  cipher_map_()
		{
			OpenSSL_add_all_algorithms();

			factory<SessionEVP> evp_factory;
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::TripleDES, CryptoEncryption::CBC), evp_factory(EVP_des_ede3_cbc()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES128, CryptoEncryption::CBC), evp_factory(EVP_aes_128_cbc()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES192, CryptoEncryption::CBC), evp_factory(EVP_aes_192_cbc()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES256, CryptoEncryption::CBC), evp_factory(EVP_aes_256_cbc()));
#if 0
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES128, CryptoEncryption::CTR), evp_factory(EVP_aes_128_ctr()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES192, CryptoEncryption::CTR), evp_factory(EVP_aes_192_ctr()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES256, CryptoEncryption::CTR), evp_factory(EVP_aes_256_ctr()));
#endif
#ifndef	OPENSSL_NO_BF
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::Blowfish, CryptoEncryption::CBC), evp_factory(EVP_bf_cbc()));
#endif
#ifndef	OPENSSL_NO_CAST
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::CAST, CryptoEncryption::CBC), evp_factory(EVP_cast5_cbc()));
#endif
#ifndef	OPENSSL_NO_IDEA
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::IDEA, CryptoEncryption::CBC), evp_factory(EVP_idea_cbc()));
#endif
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::RC4, CryptoEncryption::Stream), evp_factory(EVP_rc4()));

			/* XXX Register.  */
		}

		~MethodOpenSSL()
		{
			/* XXX Unregister.  */
		}

		std::set<CryptoEncryption::Cipher> ciphers(void) const
		{
			return (cipher_map_.keys());
		}

		CryptoEncryption::Session *session(CryptoEncryption::Cipher cipher) const
		{
			return (cipher_map_.create(cipher));
		}
	};

	static MethodOpenSSL crypto_encryption_method_openssl;
}
<commit_msg>Add support for AES-128 in CTR mode.<commit_after>#include <openssl/aes.h>
#include <openssl/evp.h>

#include <common/factory.h>

#include <crypto/crypto_encryption.h>

namespace {
	class SessionEVP : public CryptoEncryption::Session {
		LogHandle log_;
		const EVP_CIPHER *cipher_;
		EVP_CIPHER_CTX ctx_;
	public:
		SessionEVP(const EVP_CIPHER *xcipher)
		: log_("/crypto/encryption/session/openssl"),
		  cipher_(xcipher),
		  ctx_()
		{
			EVP_CIPHER_CTX_init(&ctx_);
		}

		~SessionEVP()
		{
			EVP_CIPHER_CTX_cleanup(&ctx_);
		}

		unsigned block_size(void) const
		{
			return (EVP_CIPHER_block_size(cipher_));
		}

		unsigned key_size(void) const
		{
			return (EVP_CIPHER_key_length(cipher_));
		}

		unsigned iv_size(void) const
		{
			return (EVP_CIPHER_iv_length(cipher_));
		}

		Session *clone(void) const
		{
			return (new SessionEVP(cipher_));
		}

		bool initialize(CryptoEncryption::Operation operation, const Buffer *key, const Buffer *iv)
		{
			if (key->length() < (size_t)EVP_CIPHER_key_length(cipher_))
				return (false);

			if (iv->length() < (size_t)EVP_CIPHER_iv_length(cipher_))
				return (false);

			int enc;
			switch (operation) {
			case CryptoEncryption::Encrypt:
				enc = 1;
				break;
			case CryptoEncryption::Decrypt:
				enc = 0;
				break;
			default:
				return (false);
			}

			uint8_t keydata[key->length()];
			key->copyout(keydata, sizeof keydata);

			uint8_t ivdata[iv->length()];
			iv->copyout(ivdata, sizeof ivdata);

			int rv = EVP_CipherInit(&ctx_, cipher_, keydata, ivdata, enc);
			if (rv == 0)
				return (false);

			return (true);
		}

		bool cipher(Buffer *out, const Buffer *in)
		{
			/*
			 * We process a single, large, linear byte buffer here rather
			 * than going a BufferSegment at a time, even though the byte
			 * buffer is less efficient than some alternatives, because
			 * there are padding and buffering implications if each
			 * BufferSegment's length is not modular to the block size.
			 */
			uint8_t indata[in->length()];
			in->copyout(indata, sizeof indata);

			uint8_t outdata[sizeof indata];
			int rv = EVP_Cipher(&ctx_, outdata, indata, sizeof indata);
			if (rv == 0)
				return (false);
			out->append(outdata, sizeof outdata);
			return (true);
		}

		Action *submit(Buffer *in, EventCallback *cb)
		{
			Buffer out;
			if (!cipher(&out, in)) {
				in->clear();
				cb->param(Event::Error);
				return (cb->schedule());
			}
			in->clear();
			cb->param(Event(Event::Done, out));
			return (cb->schedule());
		}
	};

	class SessionAES128CTR : public CryptoEncryption::Session {
		LogHandle log_;
		AES_KEY key_;
		uint8_t iv_[AES_BLOCK_SIZE];
	public:
		SessionAES128CTR(void)
		: log_("/crypto/encryption/session/openssl"),
		  key_(),
		  iv_()
		{ }

		~SessionAES128CTR()
		{ }

		unsigned block_size(void) const
		{
			return (AES_BLOCK_SIZE);
		}

		unsigned key_size(void) const
		{
			return (AES_BLOCK_SIZE);
		}

		unsigned iv_size(void) const
		{
			return (AES_BLOCK_SIZE);
		}

		Session *clone(void) const
		{
			return (new SessionAES128CTR());
		}

		bool initialize(CryptoEncryption::Operation operation, const Buffer *key, const Buffer *iv)
		{
			(void)operation;

			if (key->length() != AES_BLOCK_SIZE)
				return (false);

			if (iv->length() != AES_BLOCK_SIZE)
				return (false);

			uint8_t keydata[key->length()];
			key->copyout(keydata, sizeof keydata);

			AES_set_encrypt_key(keydata, AES_BLOCK_SIZE * 8, &key_);

			iv->copyout(iv_, sizeof iv_);

			return (true);
		}

		bool cipher(Buffer *out, const Buffer *in)
		{
			ASSERT(log_, in->length() % AES_BLOCK_SIZE == 0);

			/*
			 * Temporaries for AES_ctr128_encrypt.
			 *
			 * Their values only need to persist if we aren't using block-sized
			 * buffers, which we are.  We could just use AES_ctr128_inc and do
			 * the crypt operation by hand here.
			 */
			uint8_t counterbuf[AES_BLOCK_SIZE]; /* Will be initialized if countern==0.  */
			unsigned countern = 0;

			/*
			 * We process a single, large, linear byte buffer here rather
			 * than going a BufferSegment at a time, even though the byte
			 * buffer is less efficient than some alternatives, because
			 * there are padding and buffering implications if each
			 * BufferSegment's length is not modular to the block size.
			 */
			uint8_t indata[in->length()];
			in->copyout(indata, sizeof indata);

			uint8_t outdata[sizeof indata];
			AES_ctr128_encrypt(indata, outdata, sizeof indata, &key_, iv_, counterbuf, &countern);

			out->append(outdata, sizeof outdata);
			return (true);
		}

		Action *submit(Buffer *in, EventCallback *cb)
		{
			Buffer out;
			if (!cipher(&out, in)) {
				in->clear();
				cb->param(Event::Error);
				return (cb->schedule());
			}
			in->clear();
			cb->param(Event(Event::Done, out));
			return (cb->schedule());
		}
	};

	class MethodOpenSSL : public CryptoEncryption::Method {
		LogHandle log_;
		FactoryMap<CryptoEncryption::Cipher, CryptoEncryption::Session> cipher_map_;
	public:
		MethodOpenSSL(void)
		: CryptoEncryption::Method("OpenSSL"),
		  log_("/crypto/encryption/openssl"),
		  cipher_map_()
		{
			OpenSSL_add_all_algorithms();

			factory<SessionEVP> evp_factory;
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::TripleDES, CryptoEncryption::CBC), evp_factory(EVP_des_ede3_cbc()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES128, CryptoEncryption::CBC), evp_factory(EVP_aes_128_cbc()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES192, CryptoEncryption::CBC), evp_factory(EVP_aes_192_cbc()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES256, CryptoEncryption::CBC), evp_factory(EVP_aes_256_cbc()));
#if 0
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES128, CryptoEncryption::CTR), evp_factory(EVP_aes_128_ctr()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES192, CryptoEncryption::CTR), evp_factory(EVP_aes_192_ctr()));
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES256, CryptoEncryption::CTR), evp_factory(EVP_aes_256_ctr()));
#else
			factory<SessionAES128CTR> aes128ctr_factory;
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::AES128, CryptoEncryption::CTR), aes128ctr_factory());
#endif
#ifndef	OPENSSL_NO_BF
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::Blowfish, CryptoEncryption::CBC), evp_factory(EVP_bf_cbc()));
#endif
#ifndef	OPENSSL_NO_CAST
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::CAST, CryptoEncryption::CBC), evp_factory(EVP_cast5_cbc()));
#endif
#ifndef	OPENSSL_NO_IDEA
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::IDEA, CryptoEncryption::CBC), evp_factory(EVP_idea_cbc()));
#endif
			cipher_map_.enter(CryptoEncryption::Cipher(CryptoEncryption::RC4, CryptoEncryption::Stream), evp_factory(EVP_rc4()));

			/* XXX Register.  */
		}

		~MethodOpenSSL()
		{
			/* XXX Unregister.  */
		}

		std::set<CryptoEncryption::Cipher> ciphers(void) const
		{
			return (cipher_map_.keys());
		}

		CryptoEncryption::Session *session(CryptoEncryption::Cipher cipher) const
		{
			return (cipher_map_.create(cipher));
		}
	};

	static MethodOpenSSL crypto_encryption_method_openssl;
}
<|endoftext|>


<commit_before>//===--- autolink_extract_main.cpp - autolink extraction utility ----------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// Extracts autolink flags from object files so they can be passed to the
// linker directly. Mostly useful for platforms where the linker doesn't
// natively support autolinking (ie. ELF-based platforms).
//
//===----------------------------------------------------------------------===//

#include <string>
#include <vector>

#include "swift/AST/DiagnosticsFrontend.h"
#include "swift/Frontend/Frontend.h"
#include "swift/Frontend/PrintingDiagnosticConsumer.h"
#include "swift/Option/Options.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/Option.h"
#include "llvm/Object/Archive.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Object/ELFObjectFile.h"

using namespace swift;
using namespace llvm::opt;

class AutolinkExtractInvocation {
private:
  std::string MainExecutablePath;
  std::string OutputFilename = "-";
  std::vector<std::string> InputFilenames;

public:
  void setMainExecutablePath(const std::string &Path) {
    MainExecutablePath = Path;
  }

  const std::string &getOutputFilename() {
    return OutputFilename;
  }

  const std::vector<std::string> &getInputFilenames() {
    return InputFilenames;
  }

  int parseArgs(ArrayRef<const char *> Args,
                DiagnosticEngine &Diags) {
    using namespace options;

    // Parse frontend command line options using Swift's option table.
    std::unique_ptr<llvm::opt::OptTable> Table = createSwiftOptTable();
    unsigned MissingIndex;
    unsigned MissingCount;
    llvm::opt::InputArgList ParsedArgs =
      Table->ParseArgs(Args, MissingIndex, MissingCount, AutolinkExtractOption);
    if (MissingCount) {
      Diags.diagnose(SourceLoc(), diag::error_missing_arg_value,
                     ParsedArgs.getArgString(MissingIndex), MissingCount);
      return 1;
    }

    if (ParsedArgs.hasArg(OPT_UNKNOWN)) {
      for (const Arg *A : make_range(ParsedArgs.filtered_begin(OPT_UNKNOWN),
                                     ParsedArgs.filtered_end())) {
        Diags.diagnose(SourceLoc(), diag::error_unknown_arg,
                       A->getAsString(ParsedArgs));
      }
      return true;
    }

    if (ParsedArgs.getLastArg(OPT_help)) {
      std::string ExecutableName = llvm::sys::path::stem(MainExecutablePath);
      Table->PrintHelp(llvm::outs(), ExecutableName.c_str(),
                       "Swift Autolink Extract", options::AutolinkExtractOption,
                       0);
      return 1;
    }

    for (const Arg *A : make_range(ParsedArgs.filtered_begin(OPT_INPUT),
                                   ParsedArgs.filtered_end())) {
      InputFilenames.push_back(A->getValue());
    }

    if (InputFilenames.empty()) {
      Diags.diagnose(SourceLoc(), diag::error_mode_requires_an_input_file);
      return 1;
    }

    if (const Arg *A = ParsedArgs.getLastArg(OPT_o)) {
      OutputFilename = A->getValue();
    }

    return 0;
  }
};

// Look inside the binary 'Bin' and append any linker flags found in its
// ".swift1_autolink_entries" section to 'LinkerFlags'. If 'Bin' is an archive,
// recursively look inside all children within the archive. Return 'true' if
// there was an error, and 'false' otherwise.
static bool extractLinkerFlags(const llvm::object::Binary *Bin,
                               CompilerInstance &Instance,
                               StringRef BinaryFileName,
                               std::vector<std::string> &LinkerFlags) {
  if (auto *ObjectFile = llvm::dyn_cast<llvm::object::ELFObjectFileBase>(Bin)) {
    // Search for the section we hold autolink entries in
    for (auto &Section : ObjectFile->sections()) {
      llvm::StringRef SectionName;
      Section.getName(SectionName);
      if (SectionName == ".swift1_autolink_entries") {
        llvm::StringRef SectionData;
        Section.getContents(SectionData);

        // entries are null-terminated, so extract them and push them into
        // the set.
        llvm::SmallVector<llvm::StringRef, 4> SplitFlags;
        SectionData.split(SplitFlags, llvm::StringRef("\0", 1), -1,
                          /*KeepEmpty=*/false);
        for (const auto &Flag : SplitFlags)
          LinkerFlags.push_back(Flag);
      }
    }
    return false;
  } else if (auto *Archive = llvm::dyn_cast<llvm::object::Archive>(Bin)) {
    for (const auto &Child : Archive->children()) {
      auto ChildBinary = Child->getAsBinary();
      // FIXME: BinaryFileName below should instead be ld-style names for
      // object files in archives, e.g. "foo.a(bar.o)".
      if (!ChildBinary) {
        Instance.getDiags().diagnose(SourceLoc(), diag::error_open_input_file,
                                     BinaryFileName,
                                     llvm::toString(ChildBinary.takeError()));
        return true;
      }
      if (extractLinkerFlags(ChildBinary->get(), Instance, BinaryFileName,
                             LinkerFlags)) {
        return true;
      }
    }
    return false;
  } else {
    Instance.getDiags().diagnose(SourceLoc(), diag::error_open_input_file,
                                 BinaryFileName,
                                 "Don't know how to extract from object file"
                                 "format");
    return true;
  }
}

int autolink_extract_main(ArrayRef<const char *> Args, const char *Argv0,
                          void *MainAddr) {
  CompilerInstance Instance;
  PrintingDiagnosticConsumer PDC;
  Instance.addDiagnosticConsumer(&PDC);

  AutolinkExtractInvocation Invocation;
  std::string MainExecutablePath = llvm::sys::fs::getMainExecutable(Argv0,
                                                                    MainAddr);
  Invocation.setMainExecutablePath(MainExecutablePath);

  // Parse arguments.
  if (Invocation.parseArgs(Args, Instance.getDiags()) != 0) {
    return 1;
  }

  std::vector<std::string> LinkerFlags;

  // Extract the linker flags from the objects.
  for (const auto &BinaryFileName : Invocation.getInputFilenames()) {
    auto BinaryOwner = llvm::object::createBinary(BinaryFileName);
    if (!BinaryOwner) {
      std::string message;
      {
        llvm::raw_string_ostream os(message);
        logAllUnhandledErrors(BinaryOwner.takeError(), os, "");
      }

      Instance.getDiags().diagnose(SourceLoc(), diag::error_open_input_file,
                                   BinaryFileName, message);
      return 1;
    }

    if (extractLinkerFlags(BinaryOwner->getBinary(), Instance, BinaryFileName,
                           LinkerFlags)) {
      return 1;
    }
  }

  std::string OutputFilename = Invocation.getOutputFilename();
  std::error_code EC;
  llvm::raw_fd_ostream OutOS(OutputFilename, EC, llvm::sys::fs::F_None);
  if (OutOS.has_error() || EC) {
    Instance.getDiags().diagnose(SourceLoc(), diag::error_opening_output,
                                 OutputFilename, EC.message());
    OutOS.clear_error();
    return 1;
  }

  for (auto &Flag : LinkerFlags) {
    OutOS << Flag << '\n';
  }

  return 0;
}
<commit_msg>driver: adopt to new Archive API<commit_after>//===--- autolink_extract_main.cpp - autolink extraction utility ----------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// Extracts autolink flags from object files so they can be passed to the
// linker directly. Mostly useful for platforms where the linker doesn't
// natively support autolinking (ie. ELF-based platforms).
//
//===----------------------------------------------------------------------===//

#include <string>
#include <vector>

#include "swift/AST/DiagnosticsFrontend.h"
#include "swift/Frontend/Frontend.h"
#include "swift/Frontend/PrintingDiagnosticConsumer.h"
#include "swift/Option/Options.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/Option.h"
#include "llvm/Object/Archive.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Object/ELFObjectFile.h"

using namespace swift;
using namespace llvm::opt;

class AutolinkExtractInvocation {
private:
  std::string MainExecutablePath;
  std::string OutputFilename = "-";
  std::vector<std::string> InputFilenames;

public:
  void setMainExecutablePath(const std::string &Path) {
    MainExecutablePath = Path;
  }

  const std::string &getOutputFilename() {
    return OutputFilename;
  }

  const std::vector<std::string> &getInputFilenames() {
    return InputFilenames;
  }

  int parseArgs(ArrayRef<const char *> Args,
                DiagnosticEngine &Diags) {
    using namespace options;

    // Parse frontend command line options using Swift's option table.
    std::unique_ptr<llvm::opt::OptTable> Table = createSwiftOptTable();
    unsigned MissingIndex;
    unsigned MissingCount;
    llvm::opt::InputArgList ParsedArgs =
      Table->ParseArgs(Args, MissingIndex, MissingCount, AutolinkExtractOption);
    if (MissingCount) {
      Diags.diagnose(SourceLoc(), diag::error_missing_arg_value,
                     ParsedArgs.getArgString(MissingIndex), MissingCount);
      return 1;
    }

    if (ParsedArgs.hasArg(OPT_UNKNOWN)) {
      for (const Arg *A : make_range(ParsedArgs.filtered_begin(OPT_UNKNOWN),
                                     ParsedArgs.filtered_end())) {
        Diags.diagnose(SourceLoc(), diag::error_unknown_arg,
                       A->getAsString(ParsedArgs));
      }
      return true;
    }

    if (ParsedArgs.getLastArg(OPT_help)) {
      std::string ExecutableName = llvm::sys::path::stem(MainExecutablePath);
      Table->PrintHelp(llvm::outs(), ExecutableName.c_str(),
                       "Swift Autolink Extract", options::AutolinkExtractOption,
                       0);
      return 1;
    }

    for (const Arg *A : make_range(ParsedArgs.filtered_begin(OPT_INPUT),
                                   ParsedArgs.filtered_end())) {
      InputFilenames.push_back(A->getValue());
    }

    if (InputFilenames.empty()) {
      Diags.diagnose(SourceLoc(), diag::error_mode_requires_an_input_file);
      return 1;
    }

    if (const Arg *A = ParsedArgs.getLastArg(OPT_o)) {
      OutputFilename = A->getValue();
    }

    return 0;
  }
};

// Look inside the binary 'Bin' and append any linker flags found in its
// ".swift1_autolink_entries" section to 'LinkerFlags'. If 'Bin' is an archive,
// recursively look inside all children within the archive. Return 'true' if
// there was an error, and 'false' otherwise.
static bool extractLinkerFlags(const llvm::object::Binary *Bin,
                               CompilerInstance &Instance,
                               StringRef BinaryFileName,
                               std::vector<std::string> &LinkerFlags) {
  if (auto *ObjectFile = llvm::dyn_cast<llvm::object::ELFObjectFileBase>(Bin)) {
    // Search for the section we hold autolink entries in
    for (auto &Section : ObjectFile->sections()) {
      llvm::StringRef SectionName;
      Section.getName(SectionName);
      if (SectionName == ".swift1_autolink_entries") {
        llvm::StringRef SectionData;
        Section.getContents(SectionData);

        // entries are null-terminated, so extract them and push them into
        // the set.
        llvm::SmallVector<llvm::StringRef, 4> SplitFlags;
        SectionData.split(SplitFlags, llvm::StringRef("\0", 1), -1,
                          /*KeepEmpty=*/false);
        for (const auto &Flag : SplitFlags)
          LinkerFlags.push_back(Flag);
      }
    }
    return false;
  } else if (auto *Archive = llvm::dyn_cast<llvm::object::Archive>(Bin)) {
    llvm::Error Error;
    for (const auto &Child : Archive->children(Error)) {
      auto ChildBinary = Child.getAsBinary();
      // FIXME: BinaryFileName below should instead be ld-style names for
      // object files in archives, e.g. "foo.a(bar.o)".
      if (!ChildBinary) {
        Instance.getDiags().diagnose(SourceLoc(), diag::error_open_input_file,
                                     BinaryFileName,
                                     llvm::toString(ChildBinary.takeError()));
        return true;
      }
      if (extractLinkerFlags(ChildBinary->get(), Instance, BinaryFileName,
                             LinkerFlags)) {
        return true;
      }
    }
    return bool(Error);
  } else {
    Instance.getDiags().diagnose(SourceLoc(), diag::error_open_input_file,
                                 BinaryFileName,
                                 "Don't know how to extract from object file"
                                 "format");
    return true;
  }
}

int autolink_extract_main(ArrayRef<const char *> Args, const char *Argv0,
                          void *MainAddr) {
  CompilerInstance Instance;
  PrintingDiagnosticConsumer PDC;
  Instance.addDiagnosticConsumer(&PDC);

  AutolinkExtractInvocation Invocation;
  std::string MainExecutablePath = llvm::sys::fs::getMainExecutable(Argv0,
                                                                    MainAddr);
  Invocation.setMainExecutablePath(MainExecutablePath);

  // Parse arguments.
  if (Invocation.parseArgs(Args, Instance.getDiags()) != 0) {
    return 1;
  }

  std::vector<std::string> LinkerFlags;

  // Extract the linker flags from the objects.
  for (const auto &BinaryFileName : Invocation.getInputFilenames()) {
    auto BinaryOwner = llvm::object::createBinary(BinaryFileName);
    if (!BinaryOwner) {
      std::string message;
      {
        llvm::raw_string_ostream os(message);
        logAllUnhandledErrors(BinaryOwner.takeError(), os, "");
      }

      Instance.getDiags().diagnose(SourceLoc(), diag::error_open_input_file,
                                   BinaryFileName, message);
      return 1;
    }

    if (extractLinkerFlags(BinaryOwner->getBinary(), Instance, BinaryFileName,
                           LinkerFlags)) {
      return 1;
    }
  }

  std::string OutputFilename = Invocation.getOutputFilename();
  std::error_code EC;
  llvm::raw_fd_ostream OutOS(OutputFilename, EC, llvm::sys::fs::F_None);
  if (OutOS.has_error() || EC) {
    Instance.getDiags().diagnose(SourceLoc(), diag::error_opening_output,
                                 OutputFilename, EC.message());
    OutOS.clear_error();
    return 1;
  }

  for (auto &Flag : LinkerFlags) {
    OutOS << Flag << '\n';
  }

  return 0;
}
<|endoftext|>


<commit_before>#ifndef GO_SUBSAMPLEPRIMITIVES_HPP
#define GO_SUBSAMPLEPRIMITIVES_HPP

#include "globfit2/simple_types.h"
#include "globfit2/io/inputParser.hpp"
#include "globfit2/processing/util.hpp"
#include "globfit2/util/containers.hpp"
#include <chrono>

namespace globopt
{
    template <typename _Scalar>
    inline _Scalar randf() { return rand() / _Scalar(RAND_MAX); }

    // Usage: .../Release/bin/toGlobFit --subsample-primitives 0.1 --pop-limit 100 --prims segments.csv --cloud cloud.ply -a points_segments.csv --scale 0.005
    template < class _PrimitiveVectorT
             , class _PrimitiveMapT
             , class _PointContainerT
             , class _PclCloudT>
    int subsamplePrimitives( int argc, char** argv )
    {
        using namespace GF2;

        typedef typename _PrimitiveMapT::mapped_type InnerContainerT;
        typedef typename _PclCloudT::Ptr             PclPtrT;
        typedef typename InnerContainerT::value_type PrimitiveT;
        typedef typename PrimitiveT::Scalar Scalar;
        typedef typename _PointContainerT::PrimitiveT PointPrimitiveT;

        srand( time(NULL) );
        unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();

        _PointContainerT    points;
        PclPtrT             pclCloud;
        _PrimitiveVectorT   primitivesVector;
        _PrimitiveMapT      primitives;
        struct MyParams { Scalar scale; } params;

        GF2::parseInput<InnerContainerT,_PclCloudT>( points, pclCloud, primitivesVector, primitives, params, argc, argv, /* readAssoc: */ true );

        Scalar ratio( 1. );
        if ( GF2::console::parse_argument(argc,argv,"--subsample-primitives",ratio) < 0 || (ratio < FLT_EPSILON) )
        {
            std::cerr << "[" << __func__ << "]: " << "you need to provide subsample ratio after --subsample-primitives\n";
            std::cout << "example: " << "--subsample-primitives 0.1 --pop-limit 100 --prims segments.csv --cloud cloud.ply -a points_segments.csv --scale 0.005" << std::endl;
            return EXIT_FAILURE;
        }
        int popLimit( 20 ); // don't cut planes less than this
        GF2::console::parse_argument( argc,argv,"--pop-limit", popLimit );
        std::cout << "keeping at least " << popLimit << " points for each plane, change with \"--pop-limit k\"" << std::endl;

        int primLimit( 0 ); // keep the first primLimit planes
        GF2::console::parse_argument( argc,argv,"--prim-limit", primLimit );
        std::cout << "keeping first " << primLimit << " planes, change with \"--prim-limit k\"" << std::endl;

        // fetch populations
        GF2::GidPidVectorMap populations;
        processing::getPopulations( populations, points );

        //typedef typename GF2::GidPidVectorMap::value_type       Gid_PidVector_Pair;
        typedef typename GF2::GidPidVectorMap::const_iterator   PopConstIterator;
        typedef          std::pair<PidT, GidT>                  MultiMapPair;

        /// sort by population
        std::multimap< PidT, GidT > gidsBySize;         // key: #points in GId, value: { GId => [ pid0, pid1, ...] }
        PidT                        sumCanBeCut ( 0 );  // sum of #points that belong to planes, that >popLimit
        for ( PopConstIterator it = populations.begin(); it != populations.end(); ++it )
        {
            if ( it->first == PrimitiveT::LONG_VALUES::UNSET )
                continue;
            // read population
            const PidT pop = it->second.size();

            // record
            gidsBySize.insert( MultiMapPair(pop, it->first) );

            // count, how many points we plan to cut from (not all, planes < popLimit we don't touch)
            if ( pop > popLimit )
                sumCanBeCut += pop;
        } //...for populations

        /// estimate per-plane-reduction
        std::map<GidT,PidT> cutHowMuch;                             // how much each Gid has to be cut
        std::set<GidT>      keepPrims;                              // which primitives are large enough to keep. Only used, if primLimit > 0
        const PidT          targetCut( std::floor(Scalar(1.-ratio) * sumCanBeCut) ); // how many points we were instructed to cut
        PidT                sumCut   ( 0 );                         // how many points we were plan       to cut (sanity check ~= targetCut)

        std::cout << "actualLim: " << popLimit << std::endl;
        for ( auto it = gidsBySize.rbegin(); it != gidsBySize.rend(); ++it )
        {
            const PidT population = it->first;
            const GidT gid = it->second;
            //std::cout << "[#points:" << pop << "]: <gid," << gid << ">, #points:" << populations.at( it->second ).size() << std::endl;

            if ( primLimit && (keepPrims.size() >= primLimit) )
                continue;
            else
                keepPrims.insert( gid );

            if ( population < popLimit )
                continue;

            // this planes contribution to the cut
            Scalar  localRatio( Scalar(population) / Scalar(sumCanBeCut) );
            PidT    localN    ( std::floor(localRatio * targetCut) );
            if (population - localN < popLimit)
                localN = population - popLimit;

            if ( localN && (population - localN < popLimit) )
                std::cerr << "[" << __func__ << "]: " << "warning, cutting a plane to become too small: "
                          << population
                          << " --> " << population - localN
                          << " < " << popLimit
                          << std::endl;
            // record
            if ( localN )
            {
                if ( cutHowMuch.find(gid) != cutHowMuch.end() )
                    std::cerr << "[" << __func__ << "]: " << "warning, probably more planes in same Gid" << std::endl;
                cutHowMuch[ gid ] = localN;
            }

            // sanity check
            sumCut += localN;

            // log
            std::cout << "localRat = " << population << "/" << sumCanBeCut << ": " << localRatio << "\t=> * " << targetCut << " = " << localN << std::endl;
        } //...for planes to cut

        // log
        std::cout << "cut " << sumCut << "/" << points.size() << " == " << Scalar(sumCut)/Scalar(points.size()) << std::endl;

        /// do cut
        _PrimitiveMapT out;
        for ( typename _PrimitiveMapT::Iterator it(primitives); it.hasNext(); it.step() )
        {
            const GidT gid = it.getGid();

            auto popIt = populations.find( gid );
            if ( popIt == populations.end() )
            {
                std::cerr << "[" << __func__ << "]: " << " this shouldn't happen, population not found for gid " << gid << std::endl;
                continue;
            }
            auto population = popIt->second; // copy, because we reshuffle it later

            // (1) clear all point assignments, we delete the primitive
            if ( primLimit && (keepPrims.find(gid) == keepPrims.end()) )
            {
                for ( auto pidIt = population.rbegin(); pidIt != population.rend() ; ++pidIt )
                {
                    if ( points.at( /* pid: */ *pidIt ).getTag( PointPrimitiveT::TAGS::GID ) != gid )
                        std::cerr << "[" << __func__ << "]: " << "Point " << *pidIt << " is not actually assigned to plane (gid " << gid << "), should not happen" << std::endl;
                    points.at( /* pid: */ *pidIt ).setTag( PointPrimitiveT::TAGS::GID, PointPrimitiveT::LONG_VALUES::UNSET );
                }
                continue;
            }

            // store primitive
            containers::add( out, gid, *it );

            // (2) just copy primitive to output, no assigned points to remove
            auto const cutIt = cutHowMuch.find( gid );
            if ( cutIt == cutHowMuch.end() )
            {
                continue;
            }

            // (3) keep, but remove some of the assigned points
            const PidT cut = cutIt->second;

            // estimate percentage of points that need to be cut from all of the points assigned to this primitive
            Scalar  cutRatio( Scalar(cut) / Scalar( population.size()) );
            // number of points to cut
            PidT    toCut   ( std::ceil(cutRatio * Scalar(population.size())) );

            // log/debug
            if ( cutRatio < ratio * 0.01  || cutRatio > 1.01 ) std::cerr << "cutratio: " << cutRatio << std::endl;

            // reorder assigned points to randomize
            shuffle ( population.begin(), population.end(), std::default_random_engine(seed) );
            // keep track of cut points
            PidT cnt   = 0;
            // iterate through shuffled, assigned points
            for ( auto pidIt = population.rbegin(); pidIt != population.rend() && cnt != toCut; ++pidIt, ++cnt )
            {
                if ( points.at( /* pid: */ *pidIt ).getTag( PointPrimitiveT::TAGS::GID ) != gid )
                    std::cerr << "[" << __func__ << "]: " << "Point " << *pidIt << " is not actually assigned to plane (gid " << gid << "), should not happen" << std::endl;
                points.at( /* pid: */ *pidIt ).setTag( PointPrimitiveT::TAGS::GID, PointPrimitiveT::LONG_VALUES::UNSET );
            }
        } //...iterate primitives

        // check
        GF2::GidPidVectorMap populations2;
        processing::getPopulations( populations2, points );
        PidT sum(0);
        for ( auto it = populations2.begin(); it != populations2.end(); ++it )
        {
            if ( it->first == PrimitiveT::LONG_VALUES::UNSET )
                continue;

            if ( it->first < 10 )
                std::cout << "it:" << it->first << ", " << it->second.size() << ", sum: " << sum;
            sum+= it->second.size();
            if ( it->first < 10 )
                std::cout << " -> " << sum << std::endl;
        }
        std::cout << "reduction: " << points.size() << " --> " << sum << " == " << Scalar(sum) / points.size() << std::endl;

        for ( auto it = gidsBySize.begin(); it != gidsBySize.end(); ++it )
        {
            const PidT population = it->first;
            const GidT gid = it->second;
            if ( population < popLimit ) continue;
            std::cout << "oldPop[" << gid << "]: " << populations[gid].size() << ", new: " << populations2[gid].size() << std::endl;
        }

        std::string input_prims_path = parsePrimitivesPath(argc, argv);
        std::string outName = boost::filesystem::path( input_prims_path).stem().string();
        std::stringstream ssPrims; ssPrims << outName << ".sub_" << ratio << "_" << primLimit << ".csv";
        GF2::io::savePrimitives<PrimitiveT,typename InnerContainerT::const_iterator>( out, ssPrims.str() );

        std::stringstream ssAssoc; ssAssoc << "points_" << outName << ".sub_" << ratio << "_" << primLimit << ".csv";
        GF2::io::writeAssociations<PointPrimitiveT>( points, ssAssoc.str() );

        std::cout << "results written to " << ssPrims.str() << " and " << ssAssoc.str() << "\n";
        std::cout << "../runGlobfit.py -s " << params.scale << " -p " << ssPrims.str() << " -a " << ssAssoc.str() << "\n";

        return EXIT_SUCCESS;
    } //...subsamplePrimitives()
} //...ns globopt

#endif // GO_SUBSAMPLEPRIMITIVES_HPP
<commit_msg>- subsamplePrimitives now takes --prim-random-ratio to keep a certain percentage of the primitives not large, but random.<commit_after>#ifndef GO_SUBSAMPLEPRIMITIVES_HPP
#define GO_SUBSAMPLEPRIMITIVES_HPP

#include "globfit2/simple_types.h"
#include "globfit2/io/inputParser.hpp"
#include "globfit2/processing/util.hpp"
#include "globfit2/util/containers.hpp"
#include <chrono>

namespace globopt
{
    template <typename _Scalar>
    inline _Scalar randf() { return rand() / _Scalar(RAND_MAX); }

    // Usage: .../Release/bin/toGlobFit --subsample-primitives 0.1 --pop-limit 100 --prims segments.csv --cloud cloud.ply -a points_segments.csv --scale 0.005
    template < class _PrimitiveVectorT
             , class _PrimitiveMapT
             , class _PointContainerT
             , class _PclCloudT>
    int subsamplePrimitives( int argc, char** argv )
    {
        using namespace GF2;

        typedef typename _PrimitiveMapT::mapped_type    InnerContainerT;
        typedef typename _PclCloudT::Ptr                PclPtrT;
        typedef typename InnerContainerT::value_type    PrimitiveT;
        typedef typename PrimitiveT::Scalar             Scalar;
        typedef typename _PointContainerT::PrimitiveT   PointPrimitiveT;

        unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
        srand( time(NULL) );

        _PointContainerT    points;
        PclPtrT             pclCloud;
        _PrimitiveVectorT   primitivesVector;
        _PrimitiveMapT      primitives;
        struct MyParams { Scalar scale; } params;

        GF2::parseInput<InnerContainerT,_PclCloudT>( points, pclCloud, primitivesVector, primitives, params, argc, argv, /* readAssoc: */ true );

        Scalar ratio( 1. );
        if ( GF2::console::parse_argument(argc,argv,"--subsample-primitives",ratio) < 0 || (ratio < FLT_EPSILON) )
        {
            std::cerr << "[" << __func__ << "]: " << "you need to provide subsample ratio after --subsample-primitives\n";
            std::cout << "example: " << "--subsample-primitives 0.1 --prim-limit 200 --prim-random-ratio 0.5 --pop-limit 100 --prims segments.csv --cloud cloud.ply -a points_segments.csv --scale 0.005" << std::endl;
            return EXIT_FAILURE;
        }
        int popLimit( 20 ); // don't cut planes less than this
        GF2::console::parse_argument( argc,argv,"--pop-limit", popLimit );
        std::cout << "[" << __func__ << "]: " << "keeping at least " << popLimit << " points for each plane, change with \"--pop-limit k\"" << std::endl;

        int primLimit( 0 ); // keep the first primLimit planes
        GF2::console::parse_argument( argc,argv,"--prim-limit", primLimit );
        Scalar primRandRatio( 0. ); // randomize the selection of primitives
        GF2::console::parse_argument( argc,argv,"--prim-random-ratio", primRandRatio );
        std::cout << "[" << __func__ << "]: " << "keeping " << primLimit << " planes, change with \"--prim-limit k\"" << std::endl;
        std::cout << "[" << __func__ << "]: " << primRandRatio * 100.0 << "% will be random (--prim-random-ratio f), the rest picked in descending population (point count) order\n";

        // fetch populations
        GF2::GidPidVectorMap populations;
        processing::getPopulations( populations, points );

        //typedef typename GF2::GidPidVectorMap::value_type       Gid_PidVector_Pair;
        typedef typename GF2::GidPidVectorMap::const_iterator   PopConstIterator;
        typedef          std::pair<PidT, GidT>                  MultiMapPair;

        /// sort by population
        std::multimap< PidT, GidT > gidsBySize;         // key: #points in GId, value: { GId => [ pid0, pid1, ...] }
        PidT                        sumCanBeCut ( 0 );  // sum of #points that belong to planes, that >popLimit
        for ( PopConstIterator it = populations.begin(); it != populations.end(); ++it )
        {
            if ( it->first == PrimitiveT::LONG_VALUES::UNSET )
                continue;
            // read population
            const PidT pop = it->second.size();

            // record
            gidsBySize.insert( MultiMapPair(pop, it->first) );

            // count, how many points we plan to cut from (not all, planes < popLimit we don't touch)
            if ( pop > popLimit )
                sumCanBeCut += pop;
        } //...for populations

        /// estimate per-plane-reduction
        std::map<GidT,PidT> cutHowMuch;                             // how much each Gid has to be cut
        std::set<GidT>      keepPrims;                              // which primitives are large enough to keep. Only used, if primLimit > 0
        const PidT          targetCut( std::floor(Scalar(1.-ratio) * sumCanBeCut) ); // how many points we were instructed to cut
        PidT                sumCut   ( 0 );                         // how many points we were plan       to cut (sanity check ~= targetCut)

        // pick (1. - primRandRatio) primitives from the top, and the rest randomly
        std::set<MultiMapPair> chosenGids;
        {
            LidT largePrimDemand = std::floor( primLimit * (1. - primRandRatio) );
            std::cout << "largePrimDemand " << largePrimDemand << ", primlimiT: " << primLimit << std::endl;

            // iterate over all once
            auto it = gidsBySize.rbegin();
            // pick first k right away
            for ( ; it != gidsBySize.rend() && (chosenGids.size() < largePrimDemand); ++it )
                chosenGids.insert( *it );

            // enlist the rest to pick randomly from
            std::vector<MultiMapPair> gidMix; // contains the "small" primitives, to randomly sample from
            for ( ; it != gidsBySize.rend(); ++it )
                gidMix.push_back( *it );

            // shuffle the "small" primitives
            shuffle( gidMix.begin(), gidMix.end(), std::default_random_engine(seed) );

            // pick the first (primLimit-k) to fill chosen Gids
            for ( auto gidMixIt = gidMix.begin(); gidMixIt != gidMix.end() && (chosenGids.size() < primLimit); ++gidMixIt )
                chosenGids.insert( *gidMixIt );

        }

        // actually pick them, and subsample their population
        LidT primId( 0 );
        //for ( auto it = gidsBySize.rbegin(); it != gidsBySize.rend(); ++it, ++primId )
        for ( auto it = chosenGids.begin(); it != chosenGids.end(); ++it, ++primId )
        {
            const PidT population = it->first;
            const GidT gid        = it->second;
            //std::cout << "[#points:" << pop << "]: <gid," << gid << ">, #points:" << populations.at( it->second ).size() << std::endl;

            // remove primitive, if
            //  a. there's a primitive limit AND
            //  b. already exceeded that limit OR
            //  c. this is large primitive we randomly don't keep and leave space for a random one later: rand < primRandRatio
            if ( primLimit && (keepPrims.size() >= primLimit) )
            {
                std::cerr << "[" << __func__ << "]: " << " too many primitives chosen..." << std::endl;
                continue;
            }
            else
            {
                keepPrims.insert( gid );
                //std::cout << "keeping " << gid << "/" << gidsBySize.size() << std::endl;
            }

            // don't cut points from too small planes
            if ( population < popLimit )
                continue;

            // this planes contribution to the cut
            Scalar  localRatio( Scalar(population) / Scalar(sumCanBeCut) );
            PidT    localN    ( std::floor(localRatio * targetCut) );
            if ( population - localN < popLimit )
                localN = population - popLimit;

            if ( localN && (population - localN < popLimit) )
                std::cerr << "[" << __func__ << "]: " << "warning, cutting a plane to become too small: "
                          << population
                          << " --> " << population - localN
                          << " < " << popLimit
                          << std::endl;
            // record
            if ( localN )
            {
                if ( cutHowMuch.find(gid) != cutHowMuch.end() )
                    std::cerr << "[" << __func__ << "]: " << "warning, probably more planes in same Gid" << std::endl;
                cutHowMuch[ gid ] = localN;
            }

            // sanity check
            sumCut += localN;

            // log
            //std::cout << "localRat = " << population << "/" << sumCanBeCut << ": " << localRatio << "\t=> * " << targetCut << " = " << localN << std::endl;
        } //...for planes to cut

        // log
        std::cout << "[" << __func__ << "]: " << "cut " << sumCut << "/" << points.size() << " assignments == " << Scalar(sumCut)/Scalar(sumCanBeCut) * 100. << "%" << std::endl;

        /// do cut
        _PrimitiveMapT out;
        for ( typename _PrimitiveMapT::Iterator it(primitives); it.hasNext(); it.step() )
        {
            const GidT gid = it.getGid();

            auto popIt = populations.find( gid );
            if ( popIt == populations.end() )
            {
                std::cerr << "[" << __func__ << "]: " << " this shouldn't happen, population not found for gid " << gid << std::endl;
                continue;
            }
            auto population = popIt->second; // copy, because we reshuffle it later

            // (1) clear all point assignments, we delete the primitive
            if ( primLimit && (keepPrims.find(gid) == keepPrims.end()) )
            {
                for ( auto pidIt = population.rbegin(); pidIt != population.rend() ; ++pidIt )
                {
                    if ( points.at( /* pid: */ *pidIt ).getTag( PointPrimitiveT::TAGS::GID ) != gid )
                        std::cerr << "[" << __func__ << "]: " << "Point " << *pidIt << " is not actually assigned to plane (gid " << gid << "), should not happen" << std::endl;
                    points.at( /* pid: */ *pidIt ).setTag( PointPrimitiveT::TAGS::GID, PointPrimitiveT::LONG_VALUES::UNSET );
                }
                continue;
            }

            // store primitive
            containers::add( out, gid, *it );

            // (2) just copy primitive to output, no assigned points to remove
            auto const cutIt = cutHowMuch.find( gid );
            if ( cutIt == cutHowMuch.end() )
            {
                continue;
            }

            // (3) keep, but remove some of the assigned points
            const PidT cut = cutIt->second;

            // estimate percentage of points that need to be cut from all of the points assigned to this primitive
            Scalar  cutRatio( Scalar(cut) / Scalar( population.size()) );
            // number of points to cut
            PidT    toCut   ( std::ceil(cutRatio * Scalar(population.size())) );

            // log/debug
            if ( cutRatio < ratio * 0.01  || cutRatio > 1.01 ) std::cerr << "cutratio: " << cutRatio << std::endl;

            // reorder assigned points to randomize
            shuffle ( population.begin(), population.end(), std::default_random_engine(seed) );
            // keep track of cut points
            PidT cnt   = 0;
            // iterate through shuffled, assigned points
            for ( auto pidIt = population.rbegin(); pidIt != population.rend() && cnt != toCut; ++pidIt, ++cnt )
            {
                if ( points.at( /* pid: */ *pidIt ).getTag( PointPrimitiveT::TAGS::GID ) != gid )
                    std::cerr << "[" << __func__ << "]: " << "Point " << *pidIt << " is not actually assigned to plane (gid " << gid << "), should not happen" << std::endl;
                points.at( /* pid: */ *pidIt ).setTag( PointPrimitiveT::TAGS::GID, PointPrimitiveT::LONG_VALUES::UNSET );
            }
        } //...iterate primitives

        // check
        GF2::GidPidVectorMap populations2;
        processing::getPopulations( populations2, points );
        PidT                 sum(0);
        for ( auto it = populations2.begin(); it != populations2.end(); ++it )
        {
            if ( it->first == PrimitiveT::LONG_VALUES::UNSET )
                continue;

            sum+= it->second.size();
        }
        std::cout << "[" << __func__ << "]: "
                  << "Reduction: from " << points.size() << " points, and " << sumCanBeCut << " assignments --> "
                  << sum << " remaining assignments  == "
                  << Scalar(sum) / points.size() * 100. << "% points are active in the output, "
                  << Scalar(sum) / sumCanBeCut   * 100. << "% assignments remained" << std::endl;

        for ( auto it = gidsBySize.begin(); it != gidsBySize.end(); ++it )
        {
            const PidT population = it->first;
            //const GidT gid = it->second;
            if ( population < popLimit )
                continue;
            //std::cout << "oldPop[" << gid << "]: " << populations[gid].size() << ", new: " << populations2[gid].size() << std::endl;
        }

        // save
        std::string input_prims_path = parsePrimitivesPath(argc, argv);
        std::string outName = boost::filesystem::path( input_prims_path).stem().string();
        std::stringstream ssPrims; ssPrims << outName << ".sub_" << ratio << "_" << primLimit << ".csv";
        GF2::io::savePrimitives<PrimitiveT,typename InnerContainerT::const_iterator>( out, ssPrims.str() );

        std::stringstream ssAssoc; ssAssoc << "points_" << outName << ".sub_" << ratio << "_" << primLimit << ".csv";
        GF2::io::writeAssociations<PointPrimitiveT>( points, ssAssoc.str() );

        std::cout << "[" << __func__ << "]: " << "results written to " << ssPrims.str() << " and " << ssAssoc.str() << "\n";
        std::cout << "../runGlobfit.py -s " << params.scale << " -p " << ssPrims.str() << " -a " << ssAssoc.str() << "\n";

        return EXIT_SUCCESS;
    } //...subsamplePrimitives()
} //...ns globopt

#endif // GO_SUBSAMPLEPRIMITIVES_HPP
<|endoftext|>


<commit_before>//
//  LuaSystem.cpp
//  Chilli Source
//  Created by Scott Downie on 13/08/2014.
//
//  The MIT License (MIT)
//
//  Copyright (c) 2014 Tag Games Limited
//
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
//
//  The above copyright notice and this permission notice shall be included in
//  all copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
//  THE SOFTWARE.
//

#include <ChilliSource/Lua/Base/LuaSystem.h>

#include <ChilliSource/Core/Base/Utils.h>
#include <ChilliSource/Lua/Base/LuaScript.h>

extern "C"
{
#include <lua/lua.h>
#include <lua/lauxlib.h>
#include <lua/lualib.h>
}

namespace ChilliSource
{
	namespace Lua
	{
        CS_DEFINE_NAMEDTYPE(LuaSystem);
        //----------------------------------------------------
        //----------------------------------------------------
        LuaSystemUPtr LuaSystem::Create()
        {
            return LuaSystemUPtr(new LuaSystem());
        }
        //----------------------------------------------------
        //----------------------------------------------------
        bool LuaSystem::IsA(Core::InterfaceIDType in_interfaceId) const
        {
            return in_interfaceId == LuaSystem::InterfaceID;
        }
        //-------------------------------------------------------
        //-------------------------------------------------------
        void LuaSystem::OnInit()
        {
            //TODO: Open the Lua VM here when we know how to have multiple environments in
            //a single Lua state
        }
        //----------------------------------------------------
        //----------------------------------------------------
        LuaScriptUPtr LuaSystem::LoadScript(Core::StorageLocation in_location, const std::string& in_filePath)
        {
            std::string luaContents;
            Core::Utils::FileToString(in_location, in_filePath, luaContents);

            auto luaVM = luaL_newstate();
            luaL_openlibs(luaVM);

            return LuaScript::Create(luaVM, luaContents);
        }
        //----------------------------------------------------
        //----------------------------------------------------
        LuaScriptUPtr LuaSystem::CreateScript(const std::string& in_luaCode)
        {
            auto luaVM = luaL_newstate();
            luaL_openlibs(luaVM);

            return LuaScript::Create(luaVM, in_luaCode);
        }
        //-------------------------------------------------------
        //-------------------------------------------------------
        void LuaSystem::OnDestroy()
        {
            //TODO: Close the Lua VM here when we know how to have multiple environments in
            //a single Lua state
        }
	}
}
<commit_msg>Updating todo comment in LuaSystem<commit_after>//
//  LuaSystem.cpp
//  Chilli Source
//  Created by Scott Downie on 13/08/2014.
//
//  The MIT License (MIT)
//
//  Copyright (c) 2014 Tag Games Limited
//
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
//
//  The above copyright notice and this permission notice shall be included in
//  all copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
//  THE SOFTWARE.
//

#include <ChilliSource/Lua/Base/LuaSystem.h>

#include <ChilliSource/Core/Base/Utils.h>
#include <ChilliSource/Lua/Base/LuaScript.h>

extern "C"
{
#include <lua/lua.h>
#include <lua/lauxlib.h>
#include <lua/lualib.h>
}

namespace ChilliSource
{
	namespace Lua
	{
        CS_DEFINE_NAMEDTYPE(LuaSystem);
        //----------------------------------------------------
        //----------------------------------------------------
        LuaSystemUPtr LuaSystem::Create()
        {
            return LuaSystemUPtr(new LuaSystem());
        }
        //----------------------------------------------------
        //----------------------------------------------------
        bool LuaSystem::IsA(Core::InterfaceIDType in_interfaceId) const
        {
            return in_interfaceId == LuaSystem::InterfaceID;
        }
        //-------------------------------------------------------
        //-------------------------------------------------------
        void LuaSystem::OnInit()
        {
            //TODO: Open the Lua VM here when we know how to have multiple environments in
            //a single Lua state.
        }
        //----------------------------------------------------
        //----------------------------------------------------
        LuaScriptUPtr LuaSystem::LoadScript(Core::StorageLocation in_location, const std::string& in_filePath)
        {
            std::string luaContents;
            Core::Utils::FileToString(in_location, in_filePath, luaContents);

            auto luaVM = luaL_newstate();
            luaL_openlibs(luaVM);

            return LuaScript::Create(luaVM, luaContents);
        }
        //----------------------------------------------------
        //----------------------------------------------------
        LuaScriptUPtr LuaSystem::CreateScript(const std::string& in_luaCode)
        {
            auto luaVM = luaL_newstate();
            luaL_openlibs(luaVM);

            return LuaScript::Create(luaVM, in_luaCode);
        }
        //-------------------------------------------------------
        //-------------------------------------------------------
        void LuaSystem::OnDestroy()
        {
            //TODO: Close the Lua VM here when we know how to have multiple environments in
            //a single Lua state
        }
	}
}
<|endoftext|>


<commit_before>#include <algorithm>
#include "grammar_compiler.h"
#include "grammar.h"
#include "json_object.h"
#include "json_loader.h"
#include "shl_exception.h"

namespace shl {

Grammar GrammarCompiler::compile(const string& buffer) {
    JsonLoader loader;
    JsonObject object = loader.load(buffer);

    Grammar grammar;
    process(object, grammar);
    return grammar;
}

void GrammarCompiler::process(const JsonObject& object, Grammar& grammar) {
    grammar.desc = object.scope_name;
    grammar.file_types = object.file_types;

    compile_grammar(object, grammar);
    for ( auto& stocked : object.repository ) {
        const string& repo_name = stocked.first;
        const JsonObject& root = stocked.second;

        grammar.repository[repo_name] = Rule();
        compile_grammar(root, grammar.repository[repo_name]);
    }

    swap(grammar.desc, grammar.name);
}

void GrammarCompiler::compile_grammar(const JsonObject& object, Rule& rule) {
    rule.name = object.name;

    if (!object.include.empty()) {
        rule.include = WeakIncludePtr(object.include);
    } else if (!object.match.empty()) {
        rule.is_match_rule = true;
        rule.begin = Regex(object.match);
        rule.begin_captures = get_captures(object.captures);
    } else if (!object.begin.empty()) {
        rule.is_match_rule = false;
        rule.begin = Regex(object.begin);
        rule.begin_captures = get_captures(object.begin_captures);
        if (rule.begin_captures.empty()) rule.begin_captures = get_captures(object.captures);

        if(object.end.empty()) throw InvalidGrammarException("should have end for a begin/end pattern");
        rule.end = EndPatternRegex(object.end);
        rule.end_captures = get_captures(object.end_captures);
        if (rule.end_captures.empty()) rule.end_captures = get_captures(object.captures);
        rule.content_name = object.content_name;
        if (object.applyEndPatternLast == "true") rule.applyEndPatternLast = true;
    } else {
        if (object.patterns.empty()) {
            throw InvalidGrammarException("empty rule is not alowed");
        }
    }

    rule.patterns = vector<Rule>(object.patterns.size());
    for (unsigned int idx = 0; idx < rule.patterns.size(); idx++) {
        compile_grammar(object.patterns[idx], rule.patterns[idx]);
    }
}

int to_int(const string& str) {
    for (unsigned int idx = 0; idx < str.size(); idx++) {
        if (!isdigit(str[idx])) throw InvalidGrammarException("only numeric capture keys are supported");
    }
    return atoi(str.c_str());
}

string to_capture_name(const map<string, string> m) {
    auto it = m.find("name");
    if (it == m.end()) throw InvalidGrammarException("capture should have the name property");
    return it->second;
}

map<int, string> GrammarCompiler::get_captures(const map<string, map<string, string> > raw_capture) {
    map<int, string> captures;
    for(auto raw_it = raw_capture.begin(); raw_it != raw_capture.end(); raw_it++) {
        int id = to_int(raw_it->first);
        string name = to_capture_name(raw_it->second);
        captures[id] = name;
    }

    return captures;
}

static void find_include(GrammarRegistry* registry, Grammar& grammar, WeakIncludePtr& include_rule) {
    string include_name = include_rule.name;

    // base reference
    if (include_name == "$base") {
        include_rule.is_base_ref = true;

    // self reference
    } else if (include_name == "$self") {
        include_rule.ptr = &grammar;

    // repository reference
    } else if (include_name[0] == '#'){
        string repo_name = include_name.substr(1);

        auto stock_res = grammar.repository.find(repo_name);
        if (stock_res != grammar.repository.end()) {
            include_rule.ptr = &stock_res->second;
        } else {
            throw InvalidGrammarException(string("can't find the name in repository: ") + repo_name);
        }

    // external grammar reference
    } else {
        if (registry == nullptr) {
            throw ShlException("can't refer to external grammar without grammar registry");
        } else {
            //TODO report if no external grammar found
            Grammar& embeded_grammar = registry->get_grammar(include_name);
            include_rule.ptr = &embeded_grammar;
        }
    }

}

static void _resolve_include(Grammar& grammar, Rule& rule, GrammarRegistry* registry) {
    if ( rule.include.name.empty() ) {
        for ( Rule& subrule : rule.patterns ) {
            _resolve_include(grammar, subrule, registry);
        }
    } else {
        find_include(registry, grammar, rule.include);
    }
}

void GrammarCompiler::resolve_include(Grammar& grammar, GrammarRegistry* registry) {
    _resolve_include(grammar, grammar, registry);

    for ( auto& stocked_rule : grammar.repository ) {
        _resolve_include(grammar, stocked_rule.second, registry);
    }
}
}
<commit_msg>add name when error<commit_after>#include <algorithm>
#include "grammar_compiler.h"
#include "grammar.h"
#include "json_object.h"
#include "json_loader.h"
#include "shl_exception.h"

namespace shl {

Grammar GrammarCompiler::compile(const string& buffer) {
    JsonLoader loader;
    JsonObject object = loader.load(buffer);

    Grammar grammar;
    process(object, grammar);
    return grammar;
}

void GrammarCompiler::process(const JsonObject& object, Grammar& grammar) {
    grammar.desc = object.scope_name;
    grammar.file_types = object.file_types;

    compile_grammar(object, grammar);
    for ( auto& stocked : object.repository ) {
        const string& repo_name = stocked.first;
        const JsonObject& root = stocked.second;

        grammar.repository[repo_name] = Rule();
        compile_grammar(root, grammar.repository[repo_name]);
    }

    swap(grammar.desc, grammar.name);
}

void GrammarCompiler::compile_grammar(const JsonObject& object, Rule& rule) {
    rule.name = object.name;

    if (!object.include.empty()) {
        rule.include = WeakIncludePtr(object.include);
    } else if (!object.match.empty()) {
        rule.is_match_rule = true;
        rule.begin = Regex(object.match);
        rule.begin_captures = get_captures(object.captures);
    } else if (!object.begin.empty()) {
        rule.is_match_rule = false;
        rule.begin = Regex(object.begin);
        rule.begin_captures = get_captures(object.begin_captures);
        if (rule.begin_captures.empty()) rule.begin_captures = get_captures(object.captures);

        if(object.end.empty()) throw InvalidGrammarException("should have end for a begin/end pattern " + object.name);
        rule.end = EndPatternRegex(object.end);
        rule.end_captures = get_captures(object.end_captures);
        if (rule.end_captures.empty()) rule.end_captures = get_captures(object.captures);
        rule.content_name = object.content_name;
        if (object.applyEndPatternLast == "true") rule.applyEndPatternLast = true;
    } else {
        if (object.patterns.empty()) {
            throw InvalidGrammarException("empty rule is not alowed");
        }
    }

    rule.patterns = vector<Rule>(object.patterns.size());
    for (unsigned int idx = 0; idx < rule.patterns.size(); idx++) {
        compile_grammar(object.patterns[idx], rule.patterns[idx]);
    }
}

int to_int(const string& str) {
    for (unsigned int idx = 0; idx < str.size(); idx++) {
        if (!isdigit(str[idx])) throw InvalidGrammarException("only numeric capture keys are supported");
    }
    return atoi(str.c_str());
}

string to_capture_name(const map<string, string> m) {
    auto it = m.find("name");
    if (it == m.end()) throw InvalidGrammarException("capture should have the name property");
    return it->second;
}

map<int, string> GrammarCompiler::get_captures(const map<string, map<string, string> > raw_capture) {
    map<int, string> captures;
    for(auto raw_it = raw_capture.begin(); raw_it != raw_capture.end(); raw_it++) {
        int id = to_int(raw_it->first);
        string name = to_capture_name(raw_it->second);
        captures[id] = name;
    }

    return captures;
}

static void find_include(GrammarRegistry* registry, Grammar& grammar, WeakIncludePtr& include_rule) {
    string include_name = include_rule.name;

    // base reference
    if (include_name == "$base") {
        include_rule.is_base_ref = true;

    // self reference
    } else if (include_name == "$self") {
        include_rule.ptr = &grammar;

    // repository reference
    } else if (include_name[0] == '#'){
        string repo_name = include_name.substr(1);

        auto stock_res = grammar.repository.find(repo_name);
        if (stock_res != grammar.repository.end()) {
            include_rule.ptr = &stock_res->second;
        } else {
            throw InvalidGrammarException(string("can't find the name in repository: ") + repo_name);
        }

    // external grammar reference
    } else {
        if (registry == nullptr) {
            throw ShlException("can't refer to external grammar without grammar registry");
        } else {
            //TODO report if no external grammar found
            Grammar& embeded_grammar = registry->get_grammar(include_name);
            include_rule.ptr = &embeded_grammar;
        }
    }

}

static void _resolve_include(Grammar& grammar, Rule& rule, GrammarRegistry* registry) {
    if ( rule.include.name.empty() ) {
        for ( Rule& subrule : rule.patterns ) {
            _resolve_include(grammar, subrule, registry);
        }
    } else {
        find_include(registry, grammar, rule.include);
    }
}

void GrammarCompiler::resolve_include(Grammar& grammar, GrammarRegistry* registry) {
    _resolve_include(grammar, grammar, registry);

    for ( auto& stocked_rule : grammar.repository ) {
        _resolve_include(grammar, stocked_rule.second, registry);
    }
}
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 The Orbit Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "IntrospectionWindow.h"

#include "App.h"
#include "OrbitBase/Logging.h"
#include "StringManager.h"
#include "TimeGraph.h"
#include "capture_data.pb.h"

using orbit_client_model::CaptureData;
using orbit_client_protos::TimerInfo;
using orbit_grpc_protos::CaptureStarted;

IntrospectionWindow::IntrospectionWindow(OrbitApp* app) : CaptureWindow(app) {
  // Create CaptureData.
  CaptureStarted capture_started;
  capture_started.set_process_id(orbit_base::GetCurrentProcessId());
  capture_started.set_executable_path("Orbit");
  absl::flat_hash_set<uint64_t> frame_track_function_ids;
  capture_data_ = std::make_unique<CaptureData>(/*module_manager=*/nullptr, capture_started,
                                                std::move(frame_track_function_ids));
}

IntrospectionWindow::~IntrospectionWindow() { StopIntrospection(); }

const char* IntrospectionWindow::GetHelpText() const {
  const char* help_message =
      "Client Side Introspection\n\n"
      "Start/Stop Capture: 'X'\n"
      "Toggle Help: 'H'";
  return help_message;
}

bool IntrospectionWindow::IsIntrospecting() const { return introspection_listener_ != nullptr; }

void IntrospectionWindow::StartIntrospection() {
  CHECK(!IsIntrospecting());
  set_draw_help(false);
  CreateTimeGraph(capture_data_.get());
  introspection_listener_ =
      std::make_unique<orbit_base::TracingListener>([this](const orbit_base::TracingScope& scope) {
        TimerInfo timer_info;
        timer_info.set_thread_id(scope.tid);
        timer_info.set_start(scope.begin);
        timer_info.set_end(scope.end);
        timer_info.set_depth(static_cast<uint8_t>(scope.depth));
        timer_info.set_type(TimerInfo::kIntrospection);
        timer_info.mutable_registers()->Reserve(6);
        timer_info.add_registers(scope.encoded_event.args[0]);
        timer_info.add_registers(scope.encoded_event.args[1]);
        timer_info.add_registers(scope.encoded_event.args[2]);
        timer_info.add_registers(scope.encoded_event.args[3]);
        timer_info.add_registers(scope.encoded_event.args[4]);
        timer_info.add_registers(scope.encoded_event.args[5]);
        GetTimeGraph()->ProcessTimer(timer_info, /*FunctionInfo*/ nullptr);
      });
}
void IntrospectionWindow::StopIntrospection() { introspection_listener_ = nullptr; }

void IntrospectionWindow::Draw(bool viewport_was_dirty) {
  ORBIT_SCOPE_FUNCTION;
  CaptureWindow::Draw(viewport_was_dirty);
}

void IntrospectionWindow::DrawScreenSpace() {
  ORBIT_SCOPE_FUNCTION;
  CaptureWindow::DrawScreenSpace();
}

void IntrospectionWindow::RenderText(float layer) {
  ORBIT_SCOPE_FUNCTION;
  CaptureWindow::RenderText(layer);
}

void IntrospectionWindow::ToggleRecording() {
  if (!IsIntrospecting()) {
    StartIntrospection();
  } else {
    StopIntrospection();
  }
}

void IntrospectionWindow::RenderImGuiDebugUI() {
  CaptureWindow::RenderImGuiDebugUI();

  if (ImGui::CollapsingHeader("IntrospectionWindow")) {
    IMGUI_VAR_TO_TEXT(IsIntrospecting());
  }
}

bool IntrospectionWindow::ShouldAutoZoom() const { return IsIntrospecting(); }
<commit_msg>Fix the build<commit_after>// Copyright (c) 2020 The Orbit Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "IntrospectionWindow.h"

#include "App.h"
#include "OrbitBase/Logging.h"
#include "StringManager.h"
#include "TimeGraph.h"
#include "capture_data.pb.h"

using orbit_client_model::CaptureData;
using orbit_client_protos::TimerInfo;
using orbit_grpc_protos::CaptureStarted;

IntrospectionWindow::IntrospectionWindow(OrbitApp* app) : CaptureWindow(app) {
  // Create CaptureData.
  CaptureStarted capture_started;
  capture_started.set_process_id(orbit_base::GetCurrentProcessId());
  capture_started.set_executable_path("Orbit");
  absl::flat_hash_set<uint64_t> frame_track_function_ids;
  capture_data_ = std::make_unique<CaptureData>(/*module_manager=*/nullptr, capture_started,
                                                std::nullopt, std::move(frame_track_function_ids));
}

IntrospectionWindow::~IntrospectionWindow() { StopIntrospection(); }

const char* IntrospectionWindow::GetHelpText() const {
  const char* help_message =
      "Client Side Introspection\n\n"
      "Start/Stop Capture: 'X'\n"
      "Toggle Help: 'H'";
  return help_message;
}

bool IntrospectionWindow::IsIntrospecting() const { return introspection_listener_ != nullptr; }

void IntrospectionWindow::StartIntrospection() {
  CHECK(!IsIntrospecting());
  set_draw_help(false);
  CreateTimeGraph(capture_data_.get());
  introspection_listener_ =
      std::make_unique<orbit_base::TracingListener>([this](const orbit_base::TracingScope& scope) {
        TimerInfo timer_info;
        timer_info.set_thread_id(scope.tid);
        timer_info.set_start(scope.begin);
        timer_info.set_end(scope.end);
        timer_info.set_depth(static_cast<uint8_t>(scope.depth));
        timer_info.set_type(TimerInfo::kIntrospection);
        timer_info.mutable_registers()->Reserve(6);
        timer_info.add_registers(scope.encoded_event.args[0]);
        timer_info.add_registers(scope.encoded_event.args[1]);
        timer_info.add_registers(scope.encoded_event.args[2]);
        timer_info.add_registers(scope.encoded_event.args[3]);
        timer_info.add_registers(scope.encoded_event.args[4]);
        timer_info.add_registers(scope.encoded_event.args[5]);
        GetTimeGraph()->ProcessTimer(timer_info, /*FunctionInfo*/ nullptr);
      });
}
void IntrospectionWindow::StopIntrospection() { introspection_listener_ = nullptr; }

void IntrospectionWindow::Draw(bool viewport_was_dirty) {
  ORBIT_SCOPE_FUNCTION;
  CaptureWindow::Draw(viewport_was_dirty);
}

void IntrospectionWindow::DrawScreenSpace() {
  ORBIT_SCOPE_FUNCTION;
  CaptureWindow::DrawScreenSpace();
}

void IntrospectionWindow::RenderText(float layer) {
  ORBIT_SCOPE_FUNCTION;
  CaptureWindow::RenderText(layer);
}

void IntrospectionWindow::ToggleRecording() {
  if (!IsIntrospecting()) {
    StartIntrospection();
  } else {
    StopIntrospection();
  }
}

void IntrospectionWindow::RenderImGuiDebugUI() {
  CaptureWindow::RenderImGuiDebugUI();

  if (ImGui::CollapsingHeader("IntrospectionWindow")) {
    IMGUI_VAR_TO_TEXT(IsIntrospecting());
  }
}

bool IntrospectionWindow::ShouldAutoZoom() const { return IsIntrospecting(); }
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2011 Google Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include "core/inspector/PageConsoleAgent.h"

#include "bindings/core/v8/ScriptController.h"
#include "core/dom/Node.h"
#include "core/dom/NodeTraversal.h"
#include "core/dom/shadow/ShadowRoot.h"
#include "core/frame/FrameConsole.h"
#include "core/frame/FrameHost.h"
#include "core/inspector/ConsoleMessage.h"
#include "core/inspector/ConsoleMessageStorage.h"
#include "core/inspector/InjectedScriptHost.h"
#include "core/inspector/InjectedScriptManager.h"
#include "core/inspector/InspectorDOMAgent.h"
#include "core/page/Page.h"
#include "core/workers/WorkerInspectorProxy.h"

namespace blink {

int PageConsoleAgent::s_enabledAgentCount = 0;

PageConsoleAgent::PageConsoleAgent(InjectedScriptManager* injectedScriptManager, InspectorDOMAgent* domAgent, Page* page)
    : InspectorConsoleAgent(injectedScriptManager)
    , m_inspectorDOMAgent(domAgent)
    , m_page(page)
{
}

PageConsoleAgent::~PageConsoleAgent()
{
#if !ENABLE(OILPAN)
    m_inspectorDOMAgent = nullptr;
    m_instrumentingAgents->setPageConsoleAgent(nullptr);
#endif
}

void PageConsoleAgent::trace(Visitor* visitor)
{
    visitor->trace(m_inspectorDOMAgent);
    visitor->trace(m_page);
    InspectorConsoleAgent::trace(visitor);
}

void PageConsoleAgent::enable(ErrorString* errorString)
{
    InspectorConsoleAgent::enable(errorString);
    m_workersWithEnabledConsole.clear();
    m_instrumentingAgents->setPageConsoleAgent(this);
}

void PageConsoleAgent::disable(ErrorString* errorString)
{
    m_instrumentingAgents->setPageConsoleAgent(nullptr);
    InspectorConsoleAgent::disable(errorString);
}

void PageConsoleAgent::clearMessages(ErrorString* errorString)
{
    m_inspectorDOMAgent->releaseDanglingNodes();
    InspectorConsoleAgent::clearMessages(errorString);
}

void PageConsoleAgent::workerConsoleAgentEnabled(WorkerGlobalScopeProxy* proxy)
{
    m_workersWithEnabledConsole.add(proxy);
}

ConsoleMessageStorage* PageConsoleAgent::messageStorage()
{
    return &m_page->frameHost().consoleMessageStorage();
}

void PageConsoleAgent::workerTerminated(WorkerInspectorProxy* workerInspectorProxy)
{
    WorkerGlobalScopeProxy* proxy = workerInspectorProxy->workerGlobalScopeProxy();
    HashSet<WorkerGlobalScopeProxy*>::iterator iterator = m_workersWithEnabledConsole.find(proxy);
    bool workerAgentWasEnabled = iterator != m_workersWithEnabledConsole.end();
    if (workerAgentWasEnabled)
        return;

    ConsoleMessageStorage* storage = messageStorage();
    size_t messageCount = storage->size();
    for (size_t i = 0; i < messageCount; ++i) {
        ConsoleMessage* message = storage->at(i);
        if (message->workerGlobalScopeProxy() == proxy) {
            message->setWorkerGlobalScopeProxy(nullptr);
            sendConsoleMessageToFrontend(message, false);
        }
    }
}

void PageConsoleAgent::enableStackCapturingIfNeeded()
{
    if (!s_enabledAgentCount)
        ScriptController::setCaptureCallStackForUncaughtExceptions(true);
    ++s_enabledAgentCount;
}

void PageConsoleAgent::disableStackCapturingIfNeeded()
{
    if (!(--s_enabledAgentCount))
        ScriptController::setCaptureCallStackForUncaughtExceptions(false);
}

class InspectableNode final : public InjectedScriptHost::InspectableObject {
public:
    explicit InspectableNode(Node* node) : m_node(node) { }
    virtual ScriptValue get(ScriptState* state) override
    {
        return InjectedScriptHost::nodeAsScriptValue(state, m_node);
    }
private:
    Node* m_node;
};

void PageConsoleAgent::addInspectedNode(ErrorString* errorString, int nodeId)
{
    Node* node = m_inspectorDOMAgent->nodeForId(nodeId);
    if (!node) {
        *errorString = "nodeId is not valid";
        return;
    }
    while (node->isInShadowTree()) {
        Node& ancestor = NodeTraversal::highestAncestorOrSelf(*node);
        if (!ancestor.isShadowRoot() || toShadowRoot(ancestor).type() == ShadowRoot::AuthorShadowRoot)
            break;
        // User agent shadow root, keep climbing up.
        node = toShadowRoot(ancestor).host();
    }
    m_injectedScriptManager->injectedScriptHost()->addInspectedObject(adoptPtr(new InspectableNode(node)));
}

} // namespace blink
<commit_msg>[DevTools] Fix service worker crash in PageConsoleAgent<commit_after>/*
 * Copyright (C) 2011 Google Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include "core/inspector/PageConsoleAgent.h"

#include "bindings/core/v8/ScriptController.h"
#include "core/dom/Node.h"
#include "core/dom/NodeTraversal.h"
#include "core/dom/shadow/ShadowRoot.h"
#include "core/frame/FrameConsole.h"
#include "core/frame/FrameHost.h"
#include "core/inspector/ConsoleMessage.h"
#include "core/inspector/ConsoleMessageStorage.h"
#include "core/inspector/InjectedScriptHost.h"
#include "core/inspector/InjectedScriptManager.h"
#include "core/inspector/InspectorDOMAgent.h"
#include "core/page/Page.h"
#include "core/workers/WorkerInspectorProxy.h"

namespace blink {

int PageConsoleAgent::s_enabledAgentCount = 0;

PageConsoleAgent::PageConsoleAgent(InjectedScriptManager* injectedScriptManager, InspectorDOMAgent* domAgent, Page* page)
    : InspectorConsoleAgent(injectedScriptManager)
    , m_inspectorDOMAgent(domAgent)
    , m_page(page)
{
}

PageConsoleAgent::~PageConsoleAgent()
{
#if !ENABLE(OILPAN)
    m_inspectorDOMAgent = nullptr;
    m_instrumentingAgents->setPageConsoleAgent(nullptr);
#endif
}

void PageConsoleAgent::trace(Visitor* visitor)
{
    visitor->trace(m_inspectorDOMAgent);
    visitor->trace(m_page);
    InspectorConsoleAgent::trace(visitor);
}

void PageConsoleAgent::enable(ErrorString* errorString)
{
    InspectorConsoleAgent::enable(errorString);
    m_workersWithEnabledConsole.clear();
    m_instrumentingAgents->setPageConsoleAgent(this);
}

void PageConsoleAgent::disable(ErrorString* errorString)
{
    m_instrumentingAgents->setPageConsoleAgent(nullptr);
    InspectorConsoleAgent::disable(errorString);
}

void PageConsoleAgent::clearMessages(ErrorString* errorString)
{
    m_inspectorDOMAgent->releaseDanglingNodes();
    InspectorConsoleAgent::clearMessages(errorString);
}

void PageConsoleAgent::workerConsoleAgentEnabled(WorkerGlobalScopeProxy* proxy)
{
    m_workersWithEnabledConsole.add(proxy);
}

ConsoleMessageStorage* PageConsoleAgent::messageStorage()
{
    return &m_page->frameHost().consoleMessageStorage();
}

void PageConsoleAgent::workerTerminated(WorkerInspectorProxy* workerInspectorProxy)
{
    WorkerGlobalScopeProxy* proxy = workerInspectorProxy->workerGlobalScopeProxy();
    if (!proxy)
        return;

    HashSet<WorkerGlobalScopeProxy*>::iterator iterator = m_workersWithEnabledConsole.find(proxy);
    bool workerAgentWasEnabled = iterator != m_workersWithEnabledConsole.end();
    if (workerAgentWasEnabled)
        return;

    ConsoleMessageStorage* storage = messageStorage();
    size_t messageCount = storage->size();
    for (size_t i = 0; i < messageCount; ++i) {
        ConsoleMessage* message = storage->at(i);
        if (message->workerGlobalScopeProxy() == proxy) {
            message->setWorkerGlobalScopeProxy(nullptr);
            sendConsoleMessageToFrontend(message, false);
        }
    }
}

void PageConsoleAgent::enableStackCapturingIfNeeded()
{
    if (!s_enabledAgentCount)
        ScriptController::setCaptureCallStackForUncaughtExceptions(true);
    ++s_enabledAgentCount;
}

void PageConsoleAgent::disableStackCapturingIfNeeded()
{
    if (!(--s_enabledAgentCount))
        ScriptController::setCaptureCallStackForUncaughtExceptions(false);
}

class InspectableNode final : public InjectedScriptHost::InspectableObject {
public:
    explicit InspectableNode(Node* node) : m_node(node) { }
    virtual ScriptValue get(ScriptState* state) override
    {
        return InjectedScriptHost::nodeAsScriptValue(state, m_node);
    }
private:
    Node* m_node;
};

void PageConsoleAgent::addInspectedNode(ErrorString* errorString, int nodeId)
{
    Node* node = m_inspectorDOMAgent->nodeForId(nodeId);
    if (!node) {
        *errorString = "nodeId is not valid";
        return;
    }
    while (node->isInShadowTree()) {
        Node& ancestor = NodeTraversal::highestAncestorOrSelf(*node);
        if (!ancestor.isShadowRoot() || toShadowRoot(ancestor).type() == ShadowRoot::AuthorShadowRoot)
            break;
        // User agent shadow root, keep climbing up.
        node = toShadowRoot(ancestor).host();
    }
    m_injectedScriptManager->injectedScriptHost()->addInspectedObject(adoptPtr(new InspectableNode(node)));
}

} // namespace blink
<|endoftext|>