<commit_before>#pragma once

#include "generator/feature_merger.hpp"
#include "generator/generate_info.hpp"
#include "generator/popular_places_section_builder.hpp"

#include "search/utils.hpp"

#include "indexer/classificator.hpp"
#include "indexer/scales.hpp"

#include "coding/pointd_to_pointu.hpp"

#include "geometry/polygon.hpp"
#include "geometry/region2d.hpp"
#include "geometry/tree4d.hpp"

#include "base/logging.hpp"

#include <algorithm>
#include <cstdint>
#include <map>
#include <sstream>
#include <string>
#include <vector>

#include "defines.hpp"

namespace
{
class WaterBoundaryChecker
{
  uint32_t m_boundaryType;

  struct RegionTraits
  {
    m2::RectD const & LimitRect(m2::RegionD const & r) const { return r.GetRect(); }
  };
  m4::Tree<m2::RegionD, RegionTraits> m_tree;

  size_t m_totalFeatures = 0;
  size_t m_totalBorders = 0;
  size_t m_skippedBorders = 0;
  size_t m_selectedPolygons = 0;

public:
  WaterBoundaryChecker(feature::GenerateInfo const & info)
  {
    m_boundaryType = classif().GetTypeByPath({"boundary", "administrative"});
    LoadWaterGeometry(
        info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION));
  }

  ~WaterBoundaryChecker()
  {
    LOG_SHORT(LINFO, ("Features checked:", m_totalFeatures, "borders checked:", m_totalBorders,
                "borders skipped:", m_skippedBorders, "selected polygons:", m_selectedPolygons));
  }

  void LoadWaterGeometry(std::string const & rawGeometryFileName)
  {
    LOG_SHORT(LINFO, ("Loading water geometry:", rawGeometryFileName));
    FileReader reader(rawGeometryFileName);
    ReaderSource<FileReader> file(reader);

    size_t total = 0;
    while (true)
    {
      uint64_t numGeometries = 0;
      file.Read(&numGeometries, sizeof(numGeometries));

      if (numGeometries == 0)
        break;

      ++total;

      for (size_t i = 0; i < numGeometries; ++i)
      {
        uint64_t numPoints = 0;
        file.Read(&numPoints, sizeof(numPoints));

        std::vector<m2::PointD> points(numPoints);
        file.Read(points.data(), sizeof(m2::PointD) * numPoints);
        m_tree.Add(m2::RegionD(move(points)));
      }
    }
    LOG_SHORT(LINFO, ("Load", total, "water geometries"));
  }

  bool IsBoundaries(FeatureBuilder1 const & fb)
  {
    ++m_totalFeatures;

    if (fb.FindType(m_boundaryType, 2) == ftype::GetEmptyValue())
      return false;

    ++m_totalBorders;

    return true;
  }

  enum class ProcessState
  {
    Initial,
    Water,
    Earth
  };

  void ProcessBoundary(FeatureBuilder1 const & boundary, std::vector<FeatureBuilder1> & parts)
  {
    auto const & line = boundary.GetGeometry().front();

    double constexpr kExtension = 0.01;
    ProcessState state = ProcessState::Initial;

    FeatureBuilder1::PointSeq points;

    for (size_t i = 0; i < line.size(); ++i)
    {
      m2::PointD const & p = line[i];
      m2::RectD r(p.x - kExtension, p.y - kExtension, p.x + kExtension, p.y + kExtension);
      size_t hits = 0;
      m_tree.ForEachInRect(r, [&](m2::RegionD const & rgn)
      {
        ++m_selectedPolygons;
        hits += rgn.Contains(p) ? 1 : 0;
      });

      bool inWater = (hits & 0x01) == 1;

      switch (state)
      {
      case ProcessState::Initial:
      {
        if (inWater)
        {
          state = ProcessState::Water;
        }
        else
        {
          points.push_back(p);
          state = ProcessState::Earth;
        }
        break;
      }
      case ProcessState::Water:
      {
        if (inWater)
        {
          // do nothing
        }
        else
        {
          points.push_back(p);
          state = ProcessState::Earth;
        }
        break;
      }
      case ProcessState::Earth:
      {
        if (inWater)
        {
          if (points.size() > 1)
          {
            parts.push_back(boundary);
            parts.back().ResetGeometry();
            for (auto const & pt : points)
              parts.back().AddPoint(pt);
          }
          points.clear();
          state = ProcessState::Water;
        }
        else
        {
          points.push_back(p);
        }
        break;
      }
      }
    }

    if (points.size() > 1)
    {
      parts.push_back(boundary);
      parts.back().ResetGeometry();
      for (auto const & pt : points)
        parts.back().AddPoint(pt);
    }

    if (parts.empty())
      m_skippedBorders++;
  }
};
} // namespace

/// Process FeatureBuilder1 for world map. Main functions:
/// - check for visibility in world map
/// - merge linear features
template <class FeatureOutT>
class WorldMapGenerator
{
  class EmitterImpl : public FeatureEmitterIFace
  {
    FeatureOutT m_output;
    std::map<uint32_t, size_t> m_mapTypes;

  public:
    explicit EmitterImpl(feature::GenerateInfo const & info)
      : m_output(info.GetTmpFileName(WORLD_FILE_NAME))
    {
      LOG_SHORT(LINFO, ("Output World file:", info.GetTmpFileName(WORLD_FILE_NAME)));
    }

    ~EmitterImpl() override
    {
      Classificator const & c = classif();

      std::stringstream ss;
      ss << std::endl;
      for (auto const & p : m_mapTypes)
        ss << c.GetReadableObjectName(p.first) << " : " <<  p.second << std::endl;
      LOG_SHORT(LINFO, ("World types:", ss.str()));
    }

    /// This function is called after merging linear features.
    void operator()(FeatureBuilder1 const & fb) override
    {
      // do additional check for suitable size of feature
      if (NeedPushToWorld(fb) &&
          scales::IsGoodForLevel(scales::GetUpperWorldScale(), fb.GetLimitRect()))
        PushSure(fb);
    }

    void CalcStatistics(FeatureBuilder1 const & fb)
    {
      for (uint32_t type : fb.GetTypes())
        ++m_mapTypes[type];
    }

    bool NeedPushToWorld(FeatureBuilder1 const & fb) const
    {
      // GetMinFeatureDrawScale also checks suitable size for AREA features
      return (scales::GetUpperWorldScale() >= fb.GetMinFeatureDrawScale());
    }

    void PushSure(FeatureBuilder1 const & fb) { m_output(fb); }
  };

  EmitterImpl m_worldBucket;
  FeatureTypesProcessor m_typesCorrector;
  FeatureMergeProcessor m_merger;
  WaterBoundaryChecker m_boundaryChecker;
  generator::PopularPlaces m_popularPlaces;

public:
  explicit WorldMapGenerator(feature::GenerateInfo const & info)
      : m_worldBucket(info),
        m_merger(POINT_COORD_BITS - (scales::GetUpperScale() - scales::GetUpperWorldScale()) / 2),
        m_boundaryChecker(info)
  {
    // Do not strip last types for given tags,
    // for example, do not cut 'admin_level' in  'boundary-administrative-XXX'.
    char const * arr1[][3] = {{"boundary", "administrative", "2"},
                              {"boundary", "administrative", "3"},
                              {"boundary", "administrative", "4"}};

    for (size_t i = 0; i < ARRAY_SIZE(arr1); ++i)
      m_typesCorrector.SetDontNormalizeType(arr1[i]);

    char const * arr2[] = {"boundary", "administrative", "4", "state"};
    m_typesCorrector.SetDontNormalizeType(arr2);

    if (!info.m_popularPlacesFilename.empty())
      generator::LoadPopularPlaces(info.m_popularPlacesFilename, m_popularPlaces);
    else
      LOG(LWARNING, ("popular_places_data option not set. Popular atractions will not be added to World.mwm"));
  }

  void operator()(FeatureBuilder1 fb)
  {
    auto const isPopularAttraction = IsPopularAttraction(fb);

    if (!m_worldBucket.NeedPushToWorld(fb) && !isPopularAttraction)
      return;

    m_worldBucket.CalcStatistics(fb);

    if (!m_boundaryChecker.IsBoundaries(fb))
    {
      // Save original feature iff it is a popular attraction before PushFeature(fb) modifies fb.
      auto originalFeature = isPopularAttraction ? fb : FeatureBuilder1();

      if (PushFeature(fb) || !isPopularAttraction)
        return;

      // We push GEOM_POINT with all the same tags, names and center instead of GEOM_WAY/GEOM_AREA
      // because we do not need geometry for attractions (just search index and placepage data)
      // and want to avoid size checks applied to areas.
      if (originalFeature.GetGeomType() != feature::GEOM_POINT)
        originalFeature.SetCenter(originalFeature.GetGeometryCenter());

      m_worldBucket.PushSure(originalFeature);
      return;
    }

    std::vector<FeatureBuilder1> boundaryParts;
    m_boundaryChecker.ProcessBoundary(fb, boundaryParts);
    for (auto & f : boundaryParts)
      PushFeature(f);
  }

  bool PushFeature(FeatureBuilder1 & fb)
  {
    switch (fb.GetGeomType())
    {
    case feature::GEOM_LINE:
    {
      MergedFeatureBuilder1 * p = m_typesCorrector(fb);
      if (p)
        m_merger(p);
      return false;
    }
    case feature::GEOM_AREA:
    {
      // This constant is set according to size statistics.
      // Added approx 4Mb of data to the World.mwm
      auto const & geometry = fb.GetOuterGeometry();
      if (GetPolygonArea(geometry.begin(), geometry.end()) < 0.01)
        return false;
    }
    default:
      break;
    }

    if (feature::PreprocessForWorldMap(fb))
    {
      m_worldBucket.PushSure(fb);
      return true;
    }

    return false;
  }

  void DoMerge() { m_merger.DoMerge(m_worldBucket); }

private:
  bool IsPopularAttraction(FeatureBuilder1 const & fb) const
  {
    if (fb.GetName().empty())
      return false;

    auto const attractionTypes =
        search::GetCategoryTypes("attractions", "en", GetDefaultCategories());
    ASSERT(is_sorted(attractionTypes.begin(), attractionTypes.end()), ());
    auto const & featureTypes = fb.GetTypes();
    if (!std::any_of(featureTypes.begin(), featureTypes.end(), [&attractionTypes](uint32_t t) {
          return binary_search(attractionTypes.begin(), attractionTypes.end(), t);
        }))
    {
      return false;
    }

    auto const it = m_popularPlaces.find(fb.GetMostGenericOsmId());
    if (it == m_popularPlaces.end())
      return false;

    // todo(@t.yan): adjust
    uint8_t const kPopularityThreshold = 40;
    if (it->second < kPopularityThreshold)
      return false;

    // todo(@t.yan): maybe check place has wikipedia link.
    return true;
  }
};

template <class FeatureOut>
class SimpleCountryMapGenerator
{
public:
  SimpleCountryMapGenerator(feature::GenerateInfo const & info) : m_bucket(info) {}

  void operator()(FeatureBuilder1 & fb)
  {
      m_bucket(fb);
  }

  FeatureOut & Parent() { return m_bucket; }

private:
  FeatureOut m_bucket;
};

template <class FeatureOut>
class CountryMapGenerator : public SimpleCountryMapGenerator<FeatureOut>
{
public:
  CountryMapGenerator(feature::GenerateInfo const & info) :
    SimpleCountryMapGenerator<FeatureOut>(info) {}

  void operator()(FeatureBuilder1 fb)
  {
    if (feature::PreprocessForCountryMap(fb))
      SimpleCountryMapGenerator<FeatureOut>::operator()(fb);
  }
};
<commit_msg>[generator] Add international airports to World.mwm<commit_after>#pragma once

#include "generator/feature_merger.hpp"
#include "generator/generate_info.hpp"
#include "generator/popular_places_section_builder.hpp"

#include "search/utils.hpp"

#include "indexer/classificator.hpp"
#include "indexer/scales.hpp"

#include "coding/pointd_to_pointu.hpp"

#include "geometry/polygon.hpp"
#include "geometry/region2d.hpp"
#include "geometry/tree4d.hpp"

#include "base/logging.hpp"

#include <algorithm>
#include <cstdint>
#include <map>
#include <sstream>
#include <string>
#include <vector>

#include "defines.hpp"

namespace
{
class WaterBoundaryChecker
{
  uint32_t m_boundaryType;

  struct RegionTraits
  {
    m2::RectD const & LimitRect(m2::RegionD const & r) const { return r.GetRect(); }
  };
  m4::Tree<m2::RegionD, RegionTraits> m_tree;

  size_t m_totalFeatures = 0;
  size_t m_totalBorders = 0;
  size_t m_skippedBorders = 0;
  size_t m_selectedPolygons = 0;

public:
  WaterBoundaryChecker(feature::GenerateInfo const & info)
  {
    m_boundaryType = classif().GetTypeByPath({"boundary", "administrative"});
    LoadWaterGeometry(
        info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION));
  }

  ~WaterBoundaryChecker()
  {
    LOG_SHORT(LINFO, ("Features checked:", m_totalFeatures, "borders checked:", m_totalBorders,
                "borders skipped:", m_skippedBorders, "selected polygons:", m_selectedPolygons));
  }

  void LoadWaterGeometry(std::string const & rawGeometryFileName)
  {
    LOG_SHORT(LINFO, ("Loading water geometry:", rawGeometryFileName));
    FileReader reader(rawGeometryFileName);
    ReaderSource<FileReader> file(reader);

    size_t total = 0;
    while (true)
    {
      uint64_t numGeometries = 0;
      file.Read(&numGeometries, sizeof(numGeometries));

      if (numGeometries == 0)
        break;

      ++total;

      for (size_t i = 0; i < numGeometries; ++i)
      {
        uint64_t numPoints = 0;
        file.Read(&numPoints, sizeof(numPoints));

        std::vector<m2::PointD> points(numPoints);
        file.Read(points.data(), sizeof(m2::PointD) * numPoints);
        m_tree.Add(m2::RegionD(move(points)));
      }
    }
    LOG_SHORT(LINFO, ("Load", total, "water geometries"));
  }

  bool IsBoundaries(FeatureBuilder1 const & fb)
  {
    ++m_totalFeatures;

    if (fb.FindType(m_boundaryType, 2) == ftype::GetEmptyValue())
      return false;

    ++m_totalBorders;

    return true;
  }

  enum class ProcessState
  {
    Initial,
    Water,
    Earth
  };

  void ProcessBoundary(FeatureBuilder1 const & boundary, std::vector<FeatureBuilder1> & parts)
  {
    auto const & line = boundary.GetGeometry().front();

    double constexpr kExtension = 0.01;
    ProcessState state = ProcessState::Initial;

    FeatureBuilder1::PointSeq points;

    for (size_t i = 0; i < line.size(); ++i)
    {
      m2::PointD const & p = line[i];
      m2::RectD r(p.x - kExtension, p.y - kExtension, p.x + kExtension, p.y + kExtension);
      size_t hits = 0;
      m_tree.ForEachInRect(r, [&](m2::RegionD const & rgn)
      {
        ++m_selectedPolygons;
        hits += rgn.Contains(p) ? 1 : 0;
      });

      bool inWater = (hits & 0x01) == 1;

      switch (state)
      {
      case ProcessState::Initial:
      {
        if (inWater)
        {
          state = ProcessState::Water;
        }
        else
        {
          points.push_back(p);
          state = ProcessState::Earth;
        }
        break;
      }
      case ProcessState::Water:
      {
        if (inWater)
        {
          // do nothing
        }
        else
        {
          points.push_back(p);
          state = ProcessState::Earth;
        }
        break;
      }
      case ProcessState::Earth:
      {
        if (inWater)
        {
          if (points.size() > 1)
          {
            parts.push_back(boundary);
            parts.back().ResetGeometry();
            for (auto const & pt : points)
              parts.back().AddPoint(pt);
          }
          points.clear();
          state = ProcessState::Water;
        }
        else
        {
          points.push_back(p);
        }
        break;
      }
      }
    }

    if (points.size() > 1)
    {
      parts.push_back(boundary);
      parts.back().ResetGeometry();
      for (auto const & pt : points)
        parts.back().AddPoint(pt);
    }

    if (parts.empty())
      m_skippedBorders++;
  }
};
} // namespace

/// Process FeatureBuilder1 for world map. Main functions:
/// - check for visibility in world map
/// - merge linear features
template <class FeatureOutT>
class WorldMapGenerator
{
  class EmitterImpl : public FeatureEmitterIFace
  {
    FeatureOutT m_output;
    std::map<uint32_t, size_t> m_mapTypes;

  public:
    explicit EmitterImpl(feature::GenerateInfo const & info)
      : m_output(info.GetTmpFileName(WORLD_FILE_NAME))
    {
      LOG_SHORT(LINFO, ("Output World file:", info.GetTmpFileName(WORLD_FILE_NAME)));
    }

    ~EmitterImpl() override
    {
      Classificator const & c = classif();

      std::stringstream ss;
      ss << std::endl;
      for (auto const & p : m_mapTypes)
        ss << c.GetReadableObjectName(p.first) << " : " <<  p.second << std::endl;
      LOG_SHORT(LINFO, ("World types:", ss.str()));
    }

    /// This function is called after merging linear features.
    void operator()(FeatureBuilder1 const & fb) override
    {
      // do additional check for suitable size of feature
      if (NeedPushToWorld(fb) &&
          scales::IsGoodForLevel(scales::GetUpperWorldScale(), fb.GetLimitRect()))
        PushSure(fb);
    }

    void CalcStatistics(FeatureBuilder1 const & fb)
    {
      for (uint32_t type : fb.GetTypes())
        ++m_mapTypes[type];
    }

    bool NeedPushToWorld(FeatureBuilder1 const & fb) const
    {
      // GetMinFeatureDrawScale also checks suitable size for AREA features
      return (scales::GetUpperWorldScale() >= fb.GetMinFeatureDrawScale());
    }

    void PushSure(FeatureBuilder1 const & fb) { m_output(fb); }
  };

  EmitterImpl m_worldBucket;
  FeatureTypesProcessor m_typesCorrector;
  FeatureMergeProcessor m_merger;
  WaterBoundaryChecker m_boundaryChecker;
  generator::PopularPlaces m_popularPlaces;

public:
  explicit WorldMapGenerator(feature::GenerateInfo const & info)
      : m_worldBucket(info),
        m_merger(POINT_COORD_BITS - (scales::GetUpperScale() - scales::GetUpperWorldScale()) / 2),
        m_boundaryChecker(info)
  {
    // Do not strip last types for given tags,
    // for example, do not cut 'admin_level' in  'boundary-administrative-XXX'.
    char const * arr1[][3] = {{"boundary", "administrative", "2"},
                              {"boundary", "administrative", "3"},
                              {"boundary", "administrative", "4"}};

    for (size_t i = 0; i < ARRAY_SIZE(arr1); ++i)
      m_typesCorrector.SetDontNormalizeType(arr1[i]);

    char const * arr2[] = {"boundary", "administrative", "4", "state"};
    m_typesCorrector.SetDontNormalizeType(arr2);

    if (!info.m_popularPlacesFilename.empty())
      generator::LoadPopularPlaces(info.m_popularPlacesFilename, m_popularPlaces);
    else
      LOG(LWARNING, ("popular_places_data option not set. Popular atractions will not be added to World.mwm"));
  }

  void operator()(FeatureBuilder1 fb)
  {
    auto const isPopularAttraction = IsPopularAttraction(fb);
    auto const isInternationalAirport =
        fb.HasType(classif().GetTypeByPath({"aeroway", "aerodrome", "international"}));
    auto const forcePushToWorld = isPopularAttraction || isInternationalAirport;

    if (!m_worldBucket.NeedPushToWorld(fb) && !forcePushToWorld)
      return;

    m_worldBucket.CalcStatistics(fb);

    if (!m_boundaryChecker.IsBoundaries(fb))
    {
      // Save original feature iff we need to force push it before PushFeature(fb) modifies fb.
      auto originalFeature = forcePushToWorld ? fb : FeatureBuilder1();

      if (PushFeature(fb) || !forcePushToWorld)
        return;

      // We push GEOM_POINT with all the same tags, names and center instead of GEOM_WAY/GEOM_AREA
      // because we do not need geometry for invisible features (just search index and placepage
      // data) and want to avoid size checks applied to areas.
      if (originalFeature.GetGeomType() != feature::GEOM_POINT)
        originalFeature.SetCenter(originalFeature.GetGeometryCenter());

      m_worldBucket.PushSure(originalFeature);
      return;
    }

    std::vector<FeatureBuilder1> boundaryParts;
    m_boundaryChecker.ProcessBoundary(fb, boundaryParts);
    for (auto & f : boundaryParts)
      PushFeature(f);
  }

  bool PushFeature(FeatureBuilder1 & fb)
  {
    switch (fb.GetGeomType())
    {
    case feature::GEOM_LINE:
    {
      MergedFeatureBuilder1 * p = m_typesCorrector(fb);
      if (p)
        m_merger(p);
      return false;
    }
    case feature::GEOM_AREA:
    {
      // This constant is set according to size statistics.
      // Added approx 4Mb of data to the World.mwm
      auto const & geometry = fb.GetOuterGeometry();
      if (GetPolygonArea(geometry.begin(), geometry.end()) < 0.01)
        return false;
    }
    default:
      break;
    }

    if (feature::PreprocessForWorldMap(fb))
    {
      m_worldBucket.PushSure(fb);
      return true;
    }

    return false;
  }

  void DoMerge() { m_merger.DoMerge(m_worldBucket); }

private:
  bool IsPopularAttraction(FeatureBuilder1 const & fb) const
  {
    if (fb.GetName().empty())
      return false;

    auto const attractionTypes =
        search::GetCategoryTypes("attractions", "en", GetDefaultCategories());
    ASSERT(is_sorted(attractionTypes.begin(), attractionTypes.end()), ());
    auto const & featureTypes = fb.GetTypes();
    if (!std::any_of(featureTypes.begin(), featureTypes.end(), [&attractionTypes](uint32_t t) {
          return binary_search(attractionTypes.begin(), attractionTypes.end(), t);
        }))
    {
      return false;
    }

    auto const it = m_popularPlaces.find(fb.GetMostGenericOsmId());
    if (it == m_popularPlaces.end())
      return false;

    // todo(@t.yan): adjust
    uint8_t const kPopularityThreshold = 40;
    if (it->second < kPopularityThreshold)
      return false;

    // todo(@t.yan): maybe check place has wikipedia link.
    return true;
  }
};

template <class FeatureOut>
class SimpleCountryMapGenerator
{
public:
  SimpleCountryMapGenerator(feature::GenerateInfo const & info) : m_bucket(info) {}

  void operator()(FeatureBuilder1 & fb)
  {
      m_bucket(fb);
  }

  FeatureOut & Parent() { return m_bucket; }

private:
  FeatureOut m_bucket;
};

template <class FeatureOut>
class CountryMapGenerator : public SimpleCountryMapGenerator<FeatureOut>
{
public:
  CountryMapGenerator(feature::GenerateInfo const & info) :
    SimpleCountryMapGenerator<FeatureOut>(info) {}

  void operator()(FeatureBuilder1 fb)
  {
    if (feature::PreprocessForCountryMap(fb))
      SimpleCountryMapGenerator<FeatureOut>::operator()(fb);
  }
};
<|endoftext|>


<commit_before>/*

Copyright (c) 2014, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include "libtorrent/session.hpp"
#include "libtorrent/escape_string.hpp" // for from_hex
#include "libtorrent/alert_types.hpp"
#include "libtorrent/bencode.hpp" // for bencode()
#include "libtorrent/kademlia/item.hpp" // for sign_mutable_item
#include "libtorrent/ed25519.h"

#include <boost/bind.hpp>

#include <stdlib.h>

using namespace libtorrent;

#ifdef TORRENT_DISABLE_DHT

int main(int argc, char* argv[])
{
	fprintf(stderr, "not built with DHT support\n");
	return 1;
}

#else

void usage()
{
	fprintf(stderr,
		"USAGE:\ndht <command> <arg>\n\nCOMMANDS:\n"
		"get <hash>                - retrieves and prints out the immutable\n"
		"                            item stored under hash.\n"
		"put <string>              - puts the specified string as an immutable\n"
		"                            item onto the DHT. The resulting target hash\n"
		"gen-key <key-file>        - generate ed25519 keypair and save it in\n"
		"                            the specified file\n"
		"mput <key-file> <string>  - puts the specified string as a mutable\n"
		"                            object under the public key in key-file\n"
		"mget <public-key>         - get a mutable object under the specified\n"
		"                            public key\n"
		);
	exit(1);
}

std::auto_ptr<alert> wait_for_alert(session& s, int alert_type)
{
	std::auto_ptr<alert> ret;
	bool found = false;
	while (!found)
	{
		s.wait_for_alert(seconds(5));

		std::deque<alert*> alerts;
		s.pop_alerts(&alerts);
		for (std::deque<alert*>::iterator i = alerts.begin()
			, end(alerts.end()); i != end; ++i)
		{
			if ((*i)->type() != alert_type)
			{
				static int spinner = 0;
				static const char anim[] = {'-', '\\', '|', '/'};
				printf("\r%c", anim[spinner]);
				fflush(stdout);
				spinner = (spinner + 1) & 3;
				//print some alerts?
				delete *i;
				continue;
			}
			ret = std::auto_ptr<alert>(*i);
			found = true;
		}
	}
	printf("\n");
	return ret;
}

void put_string(entry& e, boost::array<char, 64>& sig, boost::uint64_t& seq
	, std::string const& salt, char const* public_key, char const* private_key
	, char const* str)
{
	using libtorrent::dht::sign_mutable_item;

	e = std::string(str);
	std::vector<char> buf;
	bencode(std::back_inserter(buf), e);
	++seq;
	sign_mutable_item(std::pair<char const*, int>(&buf[0], buf.size())
		, std::pair<char const*, int>(&salt[0], salt.size())
		, seq
		, public_key
		, private_key
		, sig.data());
}

void bootstrap(session& s)
{
	printf("bootstrapping\n");
	wait_for_alert(s, dht_bootstrap_alert::alert_type);
}

int main(int argc, char* argv[])
{
	// skip pointer to self
	++argv;
	--argc;

	if (argc < 1) usage();

	if (strcmp(argv[0], "gen-key") == 0)
	{
		++argv;
		--argc;
		if (argc < 1) usage();

		unsigned char seed[32];
		ed25519_create_seed(seed);

		FILE* f = fopen(argv[0], "wb+");
		if (f == NULL)
		{
			fprintf(stderr, "failed to open file for writing \"%s\": (%d) %s\n"
				, argv[0], errno, strerror(errno));
			return 1;
		}

		fwrite(seed, 1, 32, f);
		fclose(f);
		return 0;
	}

	session s;
	s.set_alert_mask(0xffffffff);

	s.add_dht_router(std::pair<std::string, int>("54.205.98.145", 10000));

	FILE* f = fopen(".dht", "rb");
	if (f != NULL)
	{
		fseek(f, 0, SEEK_END);
		int size = ftell(f);
		fseek(f, 0, SEEK_SET);
		if (size > 0)
		{
			std::vector<char> state;
			state.resize(size);
			fread(&state[0], 1, state.size(), f);

			lazy_entry e;
			error_code ec;
			lazy_bdecode(&state[0], &state[0] + state.size(), e, ec);
			if (ec)
				fprintf(stderr, "failed to parse .dht file: (%d) %s\n"
					, ec.value(), ec.message().c_str());
			else
				s.load_state(e);
		}
		fclose(f);
	}

	if (strcmp(argv[0], "get") == 0)
	{
		++argv;
		--argc;

		if (argc < 1) usage();

		if (strlen(argv[0]) != 40)
		{
			fprintf(stderr, "the hash is expected to be 40 hex characters\n");
			usage();
		}
		sha1_hash target;
		bool ret = from_hex(argv[0], 40, (char*)&target[0]);
		if (!ret)
		{
			fprintf(stderr, "invalid hex encoding of target hash\n");
			return 1;
		}

		bootstrap(s);
		s.dht_get_item(target);

		printf("GET %s\n", to_hex(target.to_string()).c_str());

		std::auto_ptr<alert> a = wait_for_alert(s, dht_immutable_item_alert::alert_type);

		dht_immutable_item_alert* item = alert_cast<dht_immutable_item_alert>(a.get());
		entry data;
		if (item)
			data.swap(item->item);

		printf("%s", data.to_string().c_str());
	}
	else if (strcmp(argv[0], "put") == 0)
	{
		++argv;
		--argc;
		if (argc < 1) usage();

		entry data;
		data = std::string(argv[0]);

		bootstrap(s);
		sha1_hash target = s.dht_put_item(data);

		printf("PUT %s\n", to_hex(target.to_string()).c_str());

		wait_for_alert(s, dht_put_alert::alert_type);
	}
	else if (strcmp(argv[0], "mput") == 0)
	{
		++argv;
		--argc;
		if (argc < 1) usage();

		FILE* f = fopen(argv[0], "rb+");
		if (f == NULL)
		{
			fprintf(stderr, "failed to open file \"%s\": (%d) %s\n"
				, argv[0], errno, strerror(errno));
			return 1;
		}

		unsigned char seed[32];
		fread(seed, 1, 32, f);
		fclose(f);

		++argv;
		--argc;
		if (argc < 1) usage();

		boost::array<char, 32> public_key;
		boost::array<char, 64> private_key;
		ed25519_create_keypair((unsigned char*)public_key.data()
			, (unsigned char*)private_key.data(), seed);

		bootstrap(s);
		s.dht_put_item(public_key, boost::bind(&put_string, _1, _2, _3, _4
			, public_key.data(), private_key.data(), argv[0]));

		printf("public key: %s\n", to_hex(std::string(public_key.data()
			, public_key.size())).c_str());

		wait_for_alert(s, dht_put_alert::alert_type);
	}
	else if (strcmp(argv[0], "mget") == 0)
	{
		++argv;
		--argc;
		if (argc < 1) usage();

		int len = strlen(argv[0]);
		if (len != 64)
		{
			fprintf(stderr, "public key is expected to be 64 hex digits\n");
			return 1;
		}
		boost::array<char, 32> public_key;
		bool ret = from_hex(argv[0], len, &public_key[0]);
		if (!ret)
		{
			fprintf(stderr, "invalid hex encoding of public key\n");
			return 1;
		}

		bootstrap(s);
		s.dht_get_item(public_key);

		std::auto_ptr<alert> a = wait_for_alert(s, dht_mutable_item_alert::alert_type);

		dht_mutable_item_alert* item = alert_cast<dht_mutable_item_alert>(a.get());
		entry data;
		if (item)
			data.swap(item->item);

		printf("%s", data.to_string().c_str());
	}
	else
	{
		usage();
	}

	entry e;
	s.save_state(e, session::save_dht_state);
	std::vector<char> state;
	bencode(std::back_inserter(state), e);
	f = fopen(".dht", "wb+");
	if (f == NULL)
	{
		fprintf(stderr, "failed to open file .dht for writing");
		return 1;
	}
	fwrite(&state[0], 1, state.size(), f);
	fclose(f);
}

#endif

<commit_msg>fix typo<commit_after>/*

Copyright (c) 2014, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include "libtorrent/session.hpp"
#include "libtorrent/escape_string.hpp" // for from_hex
#include "libtorrent/alert_types.hpp"
#include "libtorrent/bencode.hpp" // for bencode()
#include "libtorrent/kademlia/item.hpp" // for sign_mutable_item
#include "libtorrent/ed25519.hpp"

#include <boost/bind.hpp>

#include <stdlib.h>

using namespace libtorrent;

#ifdef TORRENT_DISABLE_DHT

int main(int argc, char* argv[])
{
	fprintf(stderr, "not built with DHT support\n");
	return 1;
}

#else

void usage()
{
	fprintf(stderr,
		"USAGE:\ndht <command> <arg>\n\nCOMMANDS:\n"
		"get <hash>                - retrieves and prints out the immutable\n"
		"                            item stored under hash.\n"
		"put <string>              - puts the specified string as an immutable\n"
		"                            item onto the DHT. The resulting target hash\n"
		"gen-key <key-file>        - generate ed25519 keypair and save it in\n"
		"                            the specified file\n"
		"mput <key-file> <string>  - puts the specified string as a mutable\n"
		"                            object under the public key in key-file\n"
		"mget <public-key>         - get a mutable object under the specified\n"
		"                            public key\n"
		);
	exit(1);
}

std::auto_ptr<alert> wait_for_alert(session& s, int alert_type)
{
	std::auto_ptr<alert> ret;
	bool found = false;
	while (!found)
	{
		s.wait_for_alert(seconds(5));

		std::deque<alert*> alerts;
		s.pop_alerts(&alerts);
		for (std::deque<alert*>::iterator i = alerts.begin()
			, end(alerts.end()); i != end; ++i)
		{
			if ((*i)->type() != alert_type)
			{
				static int spinner = 0;
				static const char anim[] = {'-', '\\', '|', '/'};
				printf("\r%c", anim[spinner]);
				fflush(stdout);
				spinner = (spinner + 1) & 3;
				//print some alerts?
				delete *i;
				continue;
			}
			ret = std::auto_ptr<alert>(*i);
			found = true;
		}
	}
	printf("\n");
	return ret;
}

void put_string(entry& e, boost::array<char, 64>& sig, boost::uint64_t& seq
	, std::string const& salt, char const* public_key, char const* private_key
	, char const* str)
{
	using libtorrent::dht::sign_mutable_item;

	e = std::string(str);
	std::vector<char> buf;
	bencode(std::back_inserter(buf), e);
	++seq;
	sign_mutable_item(std::pair<char const*, int>(&buf[0], buf.size())
		, std::pair<char const*, int>(&salt[0], salt.size())
		, seq
		, public_key
		, private_key
		, sig.data());
}

void bootstrap(session& s)
{
	printf("bootstrapping\n");
	wait_for_alert(s, dht_bootstrap_alert::alert_type);
}

int main(int argc, char* argv[])
{
	// skip pointer to self
	++argv;
	--argc;

	if (argc < 1) usage();

	if (strcmp(argv[0], "gen-key") == 0)
	{
		++argv;
		--argc;
		if (argc < 1) usage();

		unsigned char seed[32];
		ed25519_create_seed(seed);

		FILE* f = fopen(argv[0], "wb+");
		if (f == NULL)
		{
			fprintf(stderr, "failed to open file for writing \"%s\": (%d) %s\n"
				, argv[0], errno, strerror(errno));
			return 1;
		}

		fwrite(seed, 1, 32, f);
		fclose(f);
		return 0;
	}

	session s;
	s.set_alert_mask(0xffffffff);

	s.add_dht_router(std::pair<std::string, int>("54.205.98.145", 10000));

	FILE* f = fopen(".dht", "rb");
	if (f != NULL)
	{
		fseek(f, 0, SEEK_END);
		int size = ftell(f);
		fseek(f, 0, SEEK_SET);
		if (size > 0)
		{
			std::vector<char> state;
			state.resize(size);
			fread(&state[0], 1, state.size(), f);

			lazy_entry e;
			error_code ec;
			lazy_bdecode(&state[0], &state[0] + state.size(), e, ec);
			if (ec)
				fprintf(stderr, "failed to parse .dht file: (%d) %s\n"
					, ec.value(), ec.message().c_str());
			else
				s.load_state(e);
		}
		fclose(f);
	}

	if (strcmp(argv[0], "get") == 0)
	{
		++argv;
		--argc;

		if (argc < 1) usage();

		if (strlen(argv[0]) != 40)
		{
			fprintf(stderr, "the hash is expected to be 40 hex characters\n");
			usage();
		}
		sha1_hash target;
		bool ret = from_hex(argv[0], 40, (char*)&target[0]);
		if (!ret)
		{
			fprintf(stderr, "invalid hex encoding of target hash\n");
			return 1;
		}

		bootstrap(s);
		s.dht_get_item(target);

		printf("GET %s\n", to_hex(target.to_string()).c_str());

		std::auto_ptr<alert> a = wait_for_alert(s, dht_immutable_item_alert::alert_type);

		dht_immutable_item_alert* item = alert_cast<dht_immutable_item_alert>(a.get());
		entry data;
		if (item)
			data.swap(item->item);

		printf("%s", data.to_string().c_str());
	}
	else if (strcmp(argv[0], "put") == 0)
	{
		++argv;
		--argc;
		if (argc < 1) usage();

		entry data;
		data = std::string(argv[0]);

		bootstrap(s);
		sha1_hash target = s.dht_put_item(data);

		printf("PUT %s\n", to_hex(target.to_string()).c_str());

		wait_for_alert(s, dht_put_alert::alert_type);
	}
	else if (strcmp(argv[0], "mput") == 0)
	{
		++argv;
		--argc;
		if (argc < 1) usage();

		FILE* f = fopen(argv[0], "rb+");
		if (f == NULL)
		{
			fprintf(stderr, "failed to open file \"%s\": (%d) %s\n"
				, argv[0], errno, strerror(errno));
			return 1;
		}

		unsigned char seed[32];
		fread(seed, 1, 32, f);
		fclose(f);

		++argv;
		--argc;
		if (argc < 1) usage();

		boost::array<char, 32> public_key;
		boost::array<char, 64> private_key;
		ed25519_create_keypair((unsigned char*)public_key.data()
			, (unsigned char*)private_key.data(), seed);

		bootstrap(s);
		s.dht_put_item(public_key, boost::bind(&put_string, _1, _2, _3, _4
			, public_key.data(), private_key.data(), argv[0]));

		printf("public key: %s\n", to_hex(std::string(public_key.data()
			, public_key.size())).c_str());

		wait_for_alert(s, dht_put_alert::alert_type);
	}
	else if (strcmp(argv[0], "mget") == 0)
	{
		++argv;
		--argc;
		if (argc < 1) usage();

		int len = strlen(argv[0]);
		if (len != 64)
		{
			fprintf(stderr, "public key is expected to be 64 hex digits\n");
			return 1;
		}
		boost::array<char, 32> public_key;
		bool ret = from_hex(argv[0], len, &public_key[0]);
		if (!ret)
		{
			fprintf(stderr, "invalid hex encoding of public key\n");
			return 1;
		}

		bootstrap(s);
		s.dht_get_item(public_key);

		std::auto_ptr<alert> a = wait_for_alert(s, dht_mutable_item_alert::alert_type);

		dht_mutable_item_alert* item = alert_cast<dht_mutable_item_alert>(a.get());
		entry data;
		if (item)
			data.swap(item->item);

		printf("%s", data.to_string().c_str());
	}
	else
	{
		usage();
	}

	entry e;
	s.save_state(e, session::save_dht_state);
	std::vector<char> state;
	bencode(std::back_inserter(state), e);
	f = fopen(".dht", "wb+");
	if (f == NULL)
	{
		fprintf(stderr, "failed to open file .dht for writing");
		return 1;
	}
	fwrite(&state[0], 1, state.size(), f);
	fclose(f);
}

#endif

<|endoftext|>


<commit_before>#include <cmath>
#include <network.h>

namespace ESN {

    NetworkImpl::NetworkImpl( unsigned neuronCount )
        : mPotential( neuronCount )
        , mThreshold( neuronCount )
        , mResistance( neuronCount )
        , mTimeConstant( neuronCount )
        , mOutputCurrent( neuronCount )
        , mSpikeTime( neuronCount )
        , mConnection( neuronCount )
        , mConnectionWeight( neuronCount )
    {
        for ( auto & connection : mConnection )
            connection.resize( neuronCount );

        for ( auto & connectionWeight : mConnectionWeight )
            connectionWeight.resize( neuronCount );
    }

    void NetworkImpl::Step( float step )
    {
        for ( int i = 0, n = mPotential.size(); i < n; ++ i )
        {
            float inputCurrent = 0.0f;
            for ( int j = 0, nj = mConnection[i].size(); j < nj; ++ j )
                inputCurrent +=
                    mConnection[i][j] * mConnectionWeight[i][j];

            float delta = step * ( inputCurrent * mResistance[i] -
                mPotential[i] ) / mTimeConstant[i];
            mPotential[i] += delta;
        }

        for ( int i = 0, n = mPotential.size(); i < n; ++ i )
        {
            if ( mPotential[i] > mThreshold[i] )
                mSpikeTime[i] = 0.0f;

            mOutputCurrent[i] = mSpikeTime[i] *
                std::exp( 1 - mSpikeTime[i] );

            mSpikeTime[i] += step;
        }
    }

} // namespace ESN
<commit_msg>esn : NetworkImpl : fixed calculation of input current<commit_after>#include <cmath>
#include <network.h>

namespace ESN {

    NetworkImpl::NetworkImpl( unsigned neuronCount )
        : mPotential( neuronCount )
        , mThreshold( neuronCount )
        , mResistance( neuronCount )
        , mTimeConstant( neuronCount )
        , mOutputCurrent( neuronCount )
        , mSpikeTime( neuronCount )
        , mConnection( neuronCount )
        , mConnectionWeight( neuronCount )
    {
        for ( auto & connection : mConnection )
            connection.resize( neuronCount );

        for ( auto & connectionWeight : mConnectionWeight )
            connectionWeight.resize( neuronCount );
    }

    void NetworkImpl::Step( float step )
    {
        for ( int i = 0, n = mPotential.size(); i < n; ++ i )
        {
            float inputCurrent = 0.0f;
            for ( int j = 0, nj = mConnection[i].size(); j < nj; ++ j )
            {
                int inputNeuron = mConnection[i][j];
                inputCurrent += mOutputCurrent[ inputNeuron ] *
                    mConnectionWeight[i][j];
            }

            float delta = step * ( inputCurrent * mResistance[i] -
                mPotential[i] ) / mTimeConstant[i];
            mPotential[i] += delta;
        }

        for ( int i = 0, n = mPotential.size(); i < n; ++ i )
        {
            if ( mPotential[i] > mThreshold[i] )
                mSpikeTime[i] = 0.0f;

            mOutputCurrent[i] = mSpikeTime[i] *
                std::exp( 1 - mSpikeTime[i] );

            mSpikeTime[i] += step;
        }
    }

} // namespace ESN
<|endoftext|>


<commit_before>//===- lli.cpp - LLVM Interpreter / Dynamic compiler ----------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This utility provides a simple wrapper around the LLVM Execution Engines,
// which allow the direct execution of LLVM programs through a Just-In-Time
// compiler, or through an interpreter if no JIT is available for this platform.
//
//===----------------------------------------------------------------------===//

#include "llvm/LLVMContext.h"
#include "llvm/Module.h"
#include "llvm/Type.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/CodeGen/LinkAllCodegenComponents.h"
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/JITEventListener.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/IRReader.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/PluginLoader.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/TargetSelect.h"
#include <cerrno>

#ifdef __CYGWIN__
#include <cygwin/version.h>
#if defined(CYGWIN_VERSION_DLL_MAJOR) && CYGWIN_VERSION_DLL_MAJOR<1007
#define DO_NOTHING_ATEXIT 1
#endif
#endif

using namespace llvm;

namespace {
  cl::opt<std::string>
  InputFile(cl::desc("<input bitcode>"), cl::Positional, cl::init("-"));

  cl::list<std::string>
  InputArgv(cl::ConsumeAfter, cl::desc("<program arguments>..."));

  cl::opt<bool> ForceInterpreter("force-interpreter",
                                 cl::desc("Force interpretation: disable JIT"),
                                 cl::init(false));

  cl::opt<bool> UseMCJIT(
    "use-mcjit", cl::desc("Enable use of the MC-based JIT (if available)"),
    cl::init(false));

  // Determine optimization level.
  cl::opt<char>
  OptLevel("O",
           cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
                    "(default = '-O2')"),
           cl::Prefix,
           cl::ZeroOrMore,
           cl::init(' '));

  cl::opt<std::string>
  TargetTriple("mtriple", cl::desc("Override target triple for module"));

  cl::opt<std::string>
  MArch("march",
        cl::desc("Architecture to generate assembly for (see --version)"));

  cl::opt<std::string>
  MCPU("mcpu",
       cl::desc("Target a specific cpu type (-mcpu=help for details)"),
       cl::value_desc("cpu-name"),
       cl::init(""));

  cl::list<std::string>
  MAttrs("mattr",
         cl::CommaSeparated,
         cl::desc("Target specific attributes (-mattr=help for details)"),
         cl::value_desc("a1,+a2,-a3,..."));

  cl::opt<std::string>
  EntryFunc("entry-function",
            cl::desc("Specify the entry function (default = 'main') "
                     "of the executable"),
            cl::value_desc("function"),
            cl::init("main"));

  cl::opt<std::string>
  FakeArgv0("fake-argv0",
            cl::desc("Override the 'argv[0]' value passed into the executing"
                     " program"), cl::value_desc("executable"));

  cl::opt<bool>
  DisableCoreFiles("disable-core-files", cl::Hidden,
                   cl::desc("Disable emission of core files if possible"));

  cl::opt<bool>
  NoLazyCompilation("disable-lazy-compilation",
                  cl::desc("Disable JIT lazy compilation"),
                  cl::init(false));

  cl::opt<Reloc::Model>
  RelocModel("relocation-model",
             cl::desc("Choose relocation model"),
             cl::init(Reloc::Default),
             cl::values(
            clEnumValN(Reloc::Default, "default",
                       "Target default relocation model"),
            clEnumValN(Reloc::Static, "static",
                       "Non-relocatable code"),
            clEnumValN(Reloc::PIC_, "pic",
                       "Fully relocatable, position independent code"),
            clEnumValN(Reloc::DynamicNoPIC, "dynamic-no-pic",
                       "Relocatable external references, non-relocatable code"),
            clEnumValEnd));

  cl::opt<llvm::CodeModel::Model>
  CMModel("code-model",
          cl::desc("Choose code model"),
          cl::init(CodeModel::JITDefault),
          cl::values(clEnumValN(CodeModel::JITDefault, "default",
                                "Target default JIT code model"),
                     clEnumValN(CodeModel::Small, "small",
                                "Small code model"),
                     clEnumValN(CodeModel::Kernel, "kernel",
                                "Kernel code model"),
                     clEnumValN(CodeModel::Medium, "medium",
                                "Medium code model"),
                     clEnumValN(CodeModel::Large, "large",
                                "Large code model"),
                     clEnumValEnd));

}

static ExecutionEngine *EE = 0;

static void do_shutdown() {
  // Cygwin-1.5 invokes DLL's dtors before atexit handler.
#ifndef DO_NOTHING_ATEXIT
  delete EE;
  llvm_shutdown();
#endif
}

//===----------------------------------------------------------------------===//
// main Driver function
//
int main(int argc, char **argv, char * const *envp) {
  sys::PrintStackTraceOnErrorSignal();
  PrettyStackTraceProgram X(argc, argv);

  LLVMContext &Context = getGlobalContext();
  atexit(do_shutdown);  // Call llvm_shutdown() on exit.

  // If we have a native target, initialize it to ensure it is linked in and
  // usable by the JIT.
  InitializeNativeTarget();
  InitializeNativeTargetAsmPrinter();

  cl::ParseCommandLineOptions(argc, argv,
                              "llvm interpreter & dynamic compiler\n");

  // If the user doesn't want core files, disable them.
  if (DisableCoreFiles)
    sys::Process::PreventCoreFiles();

  // Load the bitcode...
  SMDiagnostic Err;
  Module *Mod = ParseIRFile(InputFile, Err, Context);
  if (!Mod) {
    Err.print(argv[0], errs());
    return 1;
  }

  // If not jitting lazily, load the whole bitcode file eagerly too.
  std::string ErrorMsg;
  if (NoLazyCompilation) {
    if (Mod->MaterializeAllPermanently(&ErrorMsg)) {
      errs() << argv[0] << ": bitcode didn't read correctly.\n";
      errs() << "Reason: " << ErrorMsg << "\n";
      exit(1);
    }
  }

  EngineBuilder builder(Mod);
  builder.setMArch(MArch);
  builder.setMCPU(MCPU);
  builder.setMAttrs(MAttrs);
  builder.setRelocationModel(RelocModel);
  builder.setCodeModel(CMModel);
  builder.setErrorStr(&ErrorMsg);
  builder.setEngineKind(ForceInterpreter
                        ? EngineKind::Interpreter
                        : EngineKind::JIT);

  // If we are supposed to override the target triple, do so now.
  if (!TargetTriple.empty())
    Mod->setTargetTriple(Triple::normalize(TargetTriple));

  // Enable MCJIT, if desired.
  if (UseMCJIT)
    builder.setUseMCJIT(true);

  CodeGenOpt::Level OLvl = CodeGenOpt::Default;
  switch (OptLevel) {
  default:
    errs() << argv[0] << ": invalid optimization level.\n";
    return 1;
  case ' ': break;
  case '0': OLvl = CodeGenOpt::None; break;
  case '1': OLvl = CodeGenOpt::Less; break;
  case '2': OLvl = CodeGenOpt::Default; break;
  case '3': OLvl = CodeGenOpt::Aggressive; break;
  }
  builder.setOptLevel(OLvl);

  EE = builder.create();
  if (!EE) {
    if (!ErrorMsg.empty())
      errs() << argv[0] << ": error creating EE: " << ErrorMsg << "\n";
    else
      errs() << argv[0] << ": unknown error creating EE!\n";
    exit(1);
  }

  EE->RegisterJITEventListener(createOProfileJITEventListener());

  EE->DisableLazyCompilation(NoLazyCompilation);

  // If the user specifically requested an argv[0] to pass into the program,
  // do it now.
  if (!FakeArgv0.empty()) {
    InputFile = FakeArgv0;
  } else {
    // Otherwise, if there is a .bc suffix on the executable strip it off, it
    // might confuse the program.
    if (StringRef(InputFile).endswith(".bc"))
      InputFile.erase(InputFile.length() - 3);
  }

  // Add the module's name to the start of the vector of arguments to main().
  InputArgv.insert(InputArgv.begin(), InputFile);

  // Call the main function from M as if its signature were:
  //   int main (int argc, char **argv, const char **envp)
  // using the contents of Args to determine argc & argv, and the contents of
  // EnvVars to determine envp.
  //
  Function *EntryFn = Mod->getFunction(EntryFunc);
  if (!EntryFn) {
    errs() << '\'' << EntryFunc << "\' function not found in module.\n";
    return -1;
  }

  // If the program doesn't explicitly call exit, we will need the Exit
  // function later on to make an explicit call, so get the function now.
  Constant *Exit = Mod->getOrInsertFunction("exit", Type::getVoidTy(Context),
                                                    Type::getInt32Ty(Context),
                                                    NULL);

  // Reset errno to zero on entry to main.
  errno = 0;

  // Run static constructors.
  EE->runStaticConstructorsDestructors(false);

  if (NoLazyCompilation) {
    for (Module::iterator I = Mod->begin(), E = Mod->end(); I != E; ++I) {
      Function *Fn = &*I;
      if (Fn != EntryFn && !Fn->isDeclaration())
        EE->getPointerToFunction(Fn);
    }
  }

  // Run main.
  int Result = EE->runFunctionAsMain(EntryFn, InputArgv, envp);

  // Run static destructors.
  EE->runStaticConstructorsDestructors(true);

  // If the program didn't call exit explicitly, we should call it now.
  // This ensures that any atexit handlers get called correctly.
  if (Function *ExitF = dyn_cast<Function>(Exit)) {
    std::vector<GenericValue> Args;
    GenericValue ResultGV;
    ResultGV.IntVal = APInt(32, Result);
    Args.push_back(ResultGV);
    EE->runFunction(ExitF, Args);
    errs() << "ERROR: exit(" << Result << ") returned!\n";
    abort();
  } else {
    errs() << "ERROR: exit defined with wrong prototype!\n";
    abort();
  }
}
<commit_msg>lli should create a JIT memory manager.<commit_after>//===- lli.cpp - LLVM Interpreter / Dynamic compiler ----------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This utility provides a simple wrapper around the LLVM Execution Engines,
// which allow the direct execution of LLVM programs through a Just-In-Time
// compiler, or through an interpreter if no JIT is available for this platform.
//
//===----------------------------------------------------------------------===//

#include "llvm/LLVMContext.h"
#include "llvm/Module.h"
#include "llvm/Type.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/CodeGen/LinkAllCodegenComponents.h"
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/JITEventListener.h"
#include "llvm/ExecutionEngine/JITMemoryManager.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/IRReader.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/PluginLoader.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/TargetSelect.h"
#include <cerrno>

#ifdef __CYGWIN__
#include <cygwin/version.h>
#if defined(CYGWIN_VERSION_DLL_MAJOR) && CYGWIN_VERSION_DLL_MAJOR<1007
#define DO_NOTHING_ATEXIT 1
#endif
#endif

using namespace llvm;

namespace {
  cl::opt<std::string>
  InputFile(cl::desc("<input bitcode>"), cl::Positional, cl::init("-"));

  cl::list<std::string>
  InputArgv(cl::ConsumeAfter, cl::desc("<program arguments>..."));

  cl::opt<bool> ForceInterpreter("force-interpreter",
                                 cl::desc("Force interpretation: disable JIT"),
                                 cl::init(false));

  cl::opt<bool> UseMCJIT(
    "use-mcjit", cl::desc("Enable use of the MC-based JIT (if available)"),
    cl::init(false));

  // Determine optimization level.
  cl::opt<char>
  OptLevel("O",
           cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
                    "(default = '-O2')"),
           cl::Prefix,
           cl::ZeroOrMore,
           cl::init(' '));

  cl::opt<std::string>
  TargetTriple("mtriple", cl::desc("Override target triple for module"));

  cl::opt<std::string>
  MArch("march",
        cl::desc("Architecture to generate assembly for (see --version)"));

  cl::opt<std::string>
  MCPU("mcpu",
       cl::desc("Target a specific cpu type (-mcpu=help for details)"),
       cl::value_desc("cpu-name"),
       cl::init(""));

  cl::list<std::string>
  MAttrs("mattr",
         cl::CommaSeparated,
         cl::desc("Target specific attributes (-mattr=help for details)"),
         cl::value_desc("a1,+a2,-a3,..."));

  cl::opt<std::string>
  EntryFunc("entry-function",
            cl::desc("Specify the entry function (default = 'main') "
                     "of the executable"),
            cl::value_desc("function"),
            cl::init("main"));

  cl::opt<std::string>
  FakeArgv0("fake-argv0",
            cl::desc("Override the 'argv[0]' value passed into the executing"
                     " program"), cl::value_desc("executable"));

  cl::opt<bool>
  DisableCoreFiles("disable-core-files", cl::Hidden,
                   cl::desc("Disable emission of core files if possible"));

  cl::opt<bool>
  NoLazyCompilation("disable-lazy-compilation",
                  cl::desc("Disable JIT lazy compilation"),
                  cl::init(false));

  cl::opt<Reloc::Model>
  RelocModel("relocation-model",
             cl::desc("Choose relocation model"),
             cl::init(Reloc::Default),
             cl::values(
            clEnumValN(Reloc::Default, "default",
                       "Target default relocation model"),
            clEnumValN(Reloc::Static, "static",
                       "Non-relocatable code"),
            clEnumValN(Reloc::PIC_, "pic",
                       "Fully relocatable, position independent code"),
            clEnumValN(Reloc::DynamicNoPIC, "dynamic-no-pic",
                       "Relocatable external references, non-relocatable code"),
            clEnumValEnd));

  cl::opt<llvm::CodeModel::Model>
  CMModel("code-model",
          cl::desc("Choose code model"),
          cl::init(CodeModel::JITDefault),
          cl::values(clEnumValN(CodeModel::JITDefault, "default",
                                "Target default JIT code model"),
                     clEnumValN(CodeModel::Small, "small",
                                "Small code model"),
                     clEnumValN(CodeModel::Kernel, "kernel",
                                "Kernel code model"),
                     clEnumValN(CodeModel::Medium, "medium",
                                "Medium code model"),
                     clEnumValN(CodeModel::Large, "large",
                                "Large code model"),
                     clEnumValEnd));

}

static ExecutionEngine *EE = 0;

static void do_shutdown() {
  // Cygwin-1.5 invokes DLL's dtors before atexit handler.
#ifndef DO_NOTHING_ATEXIT
  delete EE;
  llvm_shutdown();
#endif
}

//===----------------------------------------------------------------------===//
// main Driver function
//
int main(int argc, char **argv, char * const *envp) {
  sys::PrintStackTraceOnErrorSignal();
  PrettyStackTraceProgram X(argc, argv);

  LLVMContext &Context = getGlobalContext();
  atexit(do_shutdown);  // Call llvm_shutdown() on exit.

  // If we have a native target, initialize it to ensure it is linked in and
  // usable by the JIT.
  InitializeNativeTarget();
  InitializeNativeTargetAsmPrinter();

  cl::ParseCommandLineOptions(argc, argv,
                              "llvm interpreter & dynamic compiler\n");

  // If the user doesn't want core files, disable them.
  if (DisableCoreFiles)
    sys::Process::PreventCoreFiles();

  // Load the bitcode...
  SMDiagnostic Err;
  Module *Mod = ParseIRFile(InputFile, Err, Context);
  if (!Mod) {
    Err.print(argv[0], errs());
    return 1;
  }

  // If not jitting lazily, load the whole bitcode file eagerly too.
  std::string ErrorMsg;
  if (NoLazyCompilation) {
    if (Mod->MaterializeAllPermanently(&ErrorMsg)) {
      errs() << argv[0] << ": bitcode didn't read correctly.\n";
      errs() << "Reason: " << ErrorMsg << "\n";
      exit(1);
    }
  }

  JITMemoryManager *JMM = JITMemoryManager::CreateDefaultMemManager();

  EngineBuilder builder(Mod);
  builder.setMArch(MArch);
  builder.setMCPU(MCPU);
  builder.setMAttrs(MAttrs);
  builder.setRelocationModel(RelocModel);
  builder.setCodeModel(CMModel);
  builder.setErrorStr(&ErrorMsg);
  builder.setJITMemoryManager(JMM);
  builder.setEngineKind(ForceInterpreter
                        ? EngineKind::Interpreter
                        : EngineKind::JIT);

  // If we are supposed to override the target triple, do so now.
  if (!TargetTriple.empty())
    Mod->setTargetTriple(Triple::normalize(TargetTriple));

  // Enable MCJIT, if desired.
  if (UseMCJIT)
    builder.setUseMCJIT(true);

  CodeGenOpt::Level OLvl = CodeGenOpt::Default;
  switch (OptLevel) {
  default:
    errs() << argv[0] << ": invalid optimization level.\n";
    return 1;
  case ' ': break;
  case '0': OLvl = CodeGenOpt::None; break;
  case '1': OLvl = CodeGenOpt::Less; break;
  case '2': OLvl = CodeGenOpt::Default; break;
  case '3': OLvl = CodeGenOpt::Aggressive; break;
  }
  builder.setOptLevel(OLvl);

  EE = builder.create();
  if (!EE) {
    if (!ErrorMsg.empty())
      errs() << argv[0] << ": error creating EE: " << ErrorMsg << "\n";
    else
      errs() << argv[0] << ": unknown error creating EE!\n";
    exit(1);
  }

  EE->RegisterJITEventListener(createOProfileJITEventListener());

  EE->DisableLazyCompilation(NoLazyCompilation);

  // If the user specifically requested an argv[0] to pass into the program,
  // do it now.
  if (!FakeArgv0.empty()) {
    InputFile = FakeArgv0;
  } else {
    // Otherwise, if there is a .bc suffix on the executable strip it off, it
    // might confuse the program.
    if (StringRef(InputFile).endswith(".bc"))
      InputFile.erase(InputFile.length() - 3);
  }

  // Add the module's name to the start of the vector of arguments to main().
  InputArgv.insert(InputArgv.begin(), InputFile);

  // Call the main function from M as if its signature were:
  //   int main (int argc, char **argv, const char **envp)
  // using the contents of Args to determine argc & argv, and the contents of
  // EnvVars to determine envp.
  //
  Function *EntryFn = Mod->getFunction(EntryFunc);
  if (!EntryFn) {
    errs() << '\'' << EntryFunc << "\' function not found in module.\n";
    return -1;
  }

  // If the program doesn't explicitly call exit, we will need the Exit
  // function later on to make an explicit call, so get the function now.
  Constant *Exit = Mod->getOrInsertFunction("exit", Type::getVoidTy(Context),
                                                    Type::getInt32Ty(Context),
                                                    NULL);

  // Reset errno to zero on entry to main.
  errno = 0;

  // Run static constructors.
  EE->runStaticConstructorsDestructors(false);

  if (NoLazyCompilation) {
    for (Module::iterator I = Mod->begin(), E = Mod->end(); I != E; ++I) {
      Function *Fn = &*I;
      if (Fn != EntryFn && !Fn->isDeclaration())
        EE->getPointerToFunction(Fn);
    }
  }

  // Run main.
  int Result = EE->runFunctionAsMain(EntryFn, InputArgv, envp);

  // Run static destructors.
  EE->runStaticConstructorsDestructors(true);

  // If the program didn't call exit explicitly, we should call it now.
  // This ensures that any atexit handlers get called correctly.
  if (Function *ExitF = dyn_cast<Function>(Exit)) {
    std::vector<GenericValue> Args;
    GenericValue ResultGV;
    ResultGV.IntVal = APInt(32, Result);
    Args.push_back(ResultGV);
    EE->runFunction(ExitF, Args);
    errs() << "ERROR: exit(" << Result << ") returned!\n";
    abort();
  } else {
    errs() << "ERROR: exit defined with wrong prototype!\n";
    abort();
  }
}
<|endoftext|>


<commit_before>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "mutex.h"
#include "sandbox_impl.h"

// This is a C++ implementation of trusted_thread.cc.  Since it trusts
// the contents of the stack, it is not secure.  It is intended to be
// a reference implementation.  This code can be used as an aid to
// understanding what the real trusted thread does, since this code
// should be easier to read than assembly code.  It can also be used
// as a test bed for changes to the trusted thread.

namespace playground {

void die(const char *msg) {
  sys_write(2, msg, strlen(msg));
  sys_exit_group(1);
}

#define TO_STRING_1(x) #x
#define TO_STRING(x) TO_STRING_1(x)

#define assert(expr) { \
  if (!(expr)) die("assertion failed at " __FILE__ ":" TO_STRING(__LINE__) \
                   ": " #expr "\n"); }

// Perform a syscall given an array of syscall arguments.
extern "C" long DoSyscall(long regs[7]);
asm(
    ".pushsection .text, \"ax\", @progbits\n"
    ".global DoSyscall\n"
    "DoSyscall:\n"
#if defined(__x86_64__)
    "push %rdi\n"
    "push %rsi\n"
    "push %rdx\n"
    "push %r10\n"
    "push %r8\n"
    "push %r9\n"
    // Set up syscall arguments
    "mov 0x00(%rdi), %rax\n"
    // Skip 0x08 (%rdi): this comes last
    "mov 0x10(%rdi), %rsi\n"
    "mov 0x18(%rdi), %rdx\n"
    "mov 0x20(%rdi), %r10\n"
    "mov 0x28(%rdi), %r8\n"
    "mov 0x30(%rdi), %r9\n"
    "mov 0x08(%rdi), %rdi\n"
    "syscall\n"
    "pop %r9\n"
    "pop %r8\n"
    "pop %r10\n"
    "pop %rdx\n"
    "pop %rsi\n"
    "pop %rdi\n"
    "ret\n"
#elif defined(__i386__)
    "push %ebx\n"
    "push %ecx\n"
    "push %edx\n"
    "push %esi\n"
    "push %edi\n"
    "push %ebp\n"
    "mov 4+24(%esp), %ecx\n"
    // Set up syscall arguments
    "mov 0x00(%ecx), %eax\n"
    "mov 0x04(%ecx), %ebx\n"
    // Skip 0x08 (%ecx): this comes last
    "mov 0x0c(%ecx), %edx\n"
    "mov 0x10(%ecx), %esi\n"
    "mov 0x14(%ecx), %edi\n"
    "mov 0x18(%ecx), %ebp\n"
    "mov 0x08(%ecx), %ecx\n"
    "int $0x80\n"
    "pop %ebp\n"
    "pop %edi\n"
    "pop %esi\n"
    "pop %edx\n"
    "pop %ecx\n"
    "pop %ebx\n"
    "ret\n"
#else
#error Unsupported target platform
#endif
    ".popsection\n"
    );

void ReturnFromCloneSyscall(SecureMem::Args *secureMem) {
  // TODO(mseaborn): Call sigreturn() to unblock signals.
#if defined(__x86_64__)
  // Get stack argument that was passed to clone().
  void **stack = (void**) secureMem->rsi;
  // Account for the x86-64 red zone adjustment that our syscall
  // interceptor does when returning from the system call.
  stack = (void**) ((char*) stack - 128);
  *--stack = secureMem->ret;
  *--stack = secureMem->r15;
  *--stack = secureMem->r14;
  *--stack = secureMem->r13;
  *--stack = secureMem->r12;
  *--stack = secureMem->r11;
  *--stack = secureMem->r10;
  *--stack = secureMem->r9;
  *--stack = secureMem->r8;
  *--stack = secureMem->rdi;
  *--stack = secureMem->rsi;
  *--stack = secureMem->rdx;
  *--stack = secureMem->rcx;
  *--stack = secureMem->rbx;
  *--stack = secureMem->rbp;
  asm("mov %0, %%rsp\n"
      "pop %%rbp\n"
      "pop %%rbx\n"
      "pop %%rcx\n"
      "pop %%rdx\n"
      "pop %%rsi\n"
      "pop %%rdi\n"
      "pop %%r8\n"
      "pop %%r9\n"
      "pop %%r10\n"
      "pop %%r11\n"
      "pop %%r12\n"
      "pop %%r13\n"
      "pop %%r14\n"
      "pop %%r15\n"
      "mov $0, %%rax\n"
      "ret\n"
      : : "m"(stack));
#elif defined(__i386__)
  // Get stack argument that was passed to clone().
  void **stack = (void**) secureMem->ecx;
  *--stack = secureMem->ret;
  *--stack = secureMem->ebp;
  *--stack = secureMem->edi;
  *--stack = secureMem->esi;
  *--stack = secureMem->edx;
  *--stack = secureMem->ecx;
  *--stack = secureMem->ebx;
  asm("mov %0, %%esp\n"
      "pop %%ebx\n"
      "pop %%ecx\n"
      "pop %%edx\n"
      "pop %%esi\n"
      "pop %%edi\n"
      "pop %%ebp\n"
      "mov $0, %%eax\n"
      "ret\n"
      : : "m"(stack));
#else
#error Unsupported target platform
#endif
}

void InitCustomTLS(void *addr) {
  Sandbox::SysCalls sys;
#if defined(__x86_64__)
  int rc = sys.arch_prctl(ARCH_SET_GS, addr);
  assert(rc == 0);
#elif defined(__i386__)
  struct user_desc u;
  u.entry_number    = (typeof u.entry_number)-1;
  u.base_addr       = (long) addr;
  u.limit           = 0xfffff;
  u.seg_32bit       = 1;
  u.contents        = 0;
  u.read_exec_only  = 0;
  u.limit_in_pages  = 1;
  u.seg_not_present = 0;
  u.useable         = 1;
  int rc = sys.set_thread_area(&u);
  assert(rc == 0);
  asm volatile("movw %w0, %%fs"
               : : "q"(8 * u.entry_number + 3));
#else
#error Unsupported target platform
#endif
}

void UnlockSyscallMutex() {
  Sandbox::SysCalls sys;
  // TODO(mseaborn): Use clone() to be the same as trusted_thread.cc.
  int pid = sys.fork();
  assert(pid >= 0);
  if (pid == 0) {
    int rc = sys.mprotect(&Sandbox::syscall_mutex_, 0x1000,
                          PROT_READ | PROT_WRITE);
    assert(rc == 0);
    Mutex::unlockMutex(&Sandbox::syscall_mutex_);
    sys._exit(0);
  }
  int status;
  int rc = sys.waitpid(pid, &status, 0);
  assert(rc == pid);
  assert(status == 0);
}

// Allocate a stack that is never freed.
char *AllocateStack() {
  Sandbox::SysCalls sys;
  int stack_size = 0x1000;
#if defined(__i386__)
  void *stack = sys.mmap2(NULL, stack_size, PROT_READ | PROT_WRITE,
                          MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
#else
  void *stack = sys.mmap(NULL, stack_size, PROT_READ | PROT_WRITE,
                         MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
#endif
  assert(stack != MAP_FAILED);
  return (char *) stack + stack_size;
}

int HandleNewThread(void *arg) {
  SecureMem::Args *secureMem = (SecureMem::Args *) arg;
  CreateReferenceTrustedThread(secureMem->newSecureMem);
  ReturnFromCloneSyscall(secureMem);
  return 0;
}

struct ThreadArgs {
  SecureMem::Args *secureMem;
  int threadFd;
};

int TrustedThread(void *arg) {
  struct ThreadArgs *args = (struct ThreadArgs *) arg;
  SecureMem::Args *secureMem = args->secureMem;
  int fd = args->threadFd;
  Sandbox::SysCalls sys;

  int sequence_no = 2;
  while (1) {
    long syscall_args[7];
    memset(syscall_args, 0, sizeof(syscall_args));
    int got = sys.read(fd, syscall_args, 4);
    assert(got == 4);

    long syscall_result;
    int sysnum = syscall_args[0];
    if (sysnum == -1 || sysnum == -2) {
      // Syscall where the registers have been checked by the trusted
      // process, e.g. munmap() ranges must be OK.
      // Doesn't need extra memory region.
      assert(secureMem->sequence == sequence_no);
      sequence_no += 2;
      if (secureMem->syscallNum == __NR_exit) {
        int rc = sys.close(fd);
        assert(rc == 0);
        rc = sys.close(secureMem->threadFdPub);
        assert(rc == 0);
      }
      else if (secureMem->syscallNum == __NR_clone) {
        assert(sysnum == -2);
        // Note that HandleNewThread() does UnlockSyscallMutex() for us.
#if defined(__x86_64__)
        long clone_flags = (long) secureMem->rdi;
        int *pid_ptr = (int *) secureMem->rdx;
        int *tid_ptr = (int *) secureMem->r10;
        void *tls_info = secureMem->r8;
#elif defined(__i386__)
        long clone_flags = (long) secureMem->ebx;
        int *pid_ptr = (int *) secureMem->edx;
        void *tls_info = secureMem->esi;
        int *tid_ptr = (int *) secureMem->edi;
#else
#error Unsupported target platform
#endif
        syscall_result = sys.clone(HandleNewThread, (void *) AllocateStack(),
                                   clone_flags, (void *) secureMem,
                                   pid_ptr, tls_info, tid_ptr);
        assert(syscall_result > 0);
        int sent = sys.write(fd, &syscall_result, sizeof(syscall_result));
        assert(sent == sizeof(syscall_result));
        continue;
      }
      memcpy(syscall_args, &secureMem->syscallNum, sizeof(syscall_args));
      // We release the mutex before doing the syscall, because we
      // have now copied the arguments.
      if (sysnum == -2)
        UnlockSyscallMutex();
    }
    else if (sysnum == -3) {
      // RDTSC request.  Send back a dummy answer.
      int timestamp[2] = {0, 0};
      int sent = sys.write(fd, (char *) timestamp, sizeof(timestamp));
      assert(sent == 8);
      continue;
    }
    else {
      int rest_size = sizeof(syscall_args) - sizeof(long);
      got = sys.read(fd, &syscall_args[1], rest_size);
      assert(got == rest_size);
    }
    syscall_result = DoSyscall(syscall_args);
    int sent = sys.write(fd, &syscall_result, sizeof(syscall_result));
    assert(sent == sizeof(syscall_result));
  }
}

void CreateReferenceTrustedThread(SecureMem::Args *secureMem) {
  // We are in the nascent thread.
  Sandbox::SysCalls sys;

  int socks[2];
  int rc = sys.socketpair(AF_UNIX, SOCK_STREAM, 0, socks);
  assert(rc == 0);
  int threadFdPub = socks[0];
  int threadFd = socks[1];

  // Create trusted thread.
  // We omit CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID.
  int flags = CLONE_VM | CLONE_FS | CLONE_FILES |
    CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM;
  // Assumes that the stack grows down.
  char *stack_top = AllocateStack() - sizeof(struct ThreadArgs);
  struct ThreadArgs *thread_args = (struct ThreadArgs *) stack_top;
  thread_args->threadFd = threadFd;
  thread_args->secureMem = secureMem;
  rc = sys.clone(TrustedThread, stack_top, flags, thread_args,
                 NULL, NULL, NULL);
  assert(rc > 0);

  // Make the thread state pages usable.
  rc = sys.mprotect(secureMem, 0x1000, PROT_READ);
  assert(rc == 0);
  rc = sys.mprotect(((char *) secureMem) + 0x1000, 0x1000,
                    PROT_READ | PROT_WRITE);
  assert(rc == 0);

  // Using cookie as the start is a little odd because other code in
  // syscall.c works around this when addressing from %fs.
  void *tls = (void*) &secureMem->cookie;
  InitCustomTLS(tls);

  UnlockSyscallMutex();

  int tid = sys.gettid();
  assert(tid > 0);

  // Send the socket FDs to the trusted process.
  // TODO(mseaborn): Don't duplicate this struct in trusted_process.cc
  struct {
    SecureMem::Args* self;
    int              tid;
    int              fdPub;
  } __attribute__((packed)) data;
  data.self = secureMem;
  data.tid = tid;
  data.fdPub = threadFdPub;
  bool ok = Sandbox::sendFd(Sandbox::cloneFdPub_, threadFdPub, threadFd,
                            (void *) &data, sizeof(data));
  assert(ok);

  // Wait for the trusted process to fill out the thread state for us.
  char byte_message;
  int got = sys.read(threadFdPub, &byte_message, 1);
  assert(got == 1);
  assert(byte_message == 0);

  // Switch to seccomp mode.
  rc = sys.prctl(PR_SET_SECCOMP, 1);
  assert(rc == 0);
}

}
<commit_msg>Fix warning produced by gcc 4.4.1 (from Ubuntu Karmic)<commit_after>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "mutex.h"
#include "sandbox_impl.h"

// This is a C++ implementation of trusted_thread.cc.  Since it trusts
// the contents of the stack, it is not secure.  It is intended to be
// a reference implementation.  This code can be used as an aid to
// understanding what the real trusted thread does, since this code
// should be easier to read than assembly code.  It can also be used
// as a test bed for changes to the trusted thread.

namespace playground {

void die(const char *msg) {
  sys_write(2, msg, strlen(msg));
  sys_exit_group(1);
}

#define TO_STRING_1(x) #x
#define TO_STRING(x) TO_STRING_1(x)

#define assert(expr) { \
  if (!(expr)) die("assertion failed at " __FILE__ ":" TO_STRING(__LINE__) \
                   ": " #expr "\n"); }

// Perform a syscall given an array of syscall arguments.
extern "C" long DoSyscall(long regs[7]);
asm(
    ".pushsection .text, \"ax\", @progbits\n"
    ".global DoSyscall\n"
    "DoSyscall:\n"
#if defined(__x86_64__)
    "push %rdi\n"
    "push %rsi\n"
    "push %rdx\n"
    "push %r10\n"
    "push %r8\n"
    "push %r9\n"
    // Set up syscall arguments
    "mov 0x00(%rdi), %rax\n"
    // Skip 0x08 (%rdi): this comes last
    "mov 0x10(%rdi), %rsi\n"
    "mov 0x18(%rdi), %rdx\n"
    "mov 0x20(%rdi), %r10\n"
    "mov 0x28(%rdi), %r8\n"
    "mov 0x30(%rdi), %r9\n"
    "mov 0x08(%rdi), %rdi\n"
    "syscall\n"
    "pop %r9\n"
    "pop %r8\n"
    "pop %r10\n"
    "pop %rdx\n"
    "pop %rsi\n"
    "pop %rdi\n"
    "ret\n"
#elif defined(__i386__)
    "push %ebx\n"
    "push %ecx\n"
    "push %edx\n"
    "push %esi\n"
    "push %edi\n"
    "push %ebp\n"
    "mov 4+24(%esp), %ecx\n"
    // Set up syscall arguments
    "mov 0x00(%ecx), %eax\n"
    "mov 0x04(%ecx), %ebx\n"
    // Skip 0x08 (%ecx): this comes last
    "mov 0x0c(%ecx), %edx\n"
    "mov 0x10(%ecx), %esi\n"
    "mov 0x14(%ecx), %edi\n"
    "mov 0x18(%ecx), %ebp\n"
    "mov 0x08(%ecx), %ecx\n"
    "int $0x80\n"
    "pop %ebp\n"
    "pop %edi\n"
    "pop %esi\n"
    "pop %edx\n"
    "pop %ecx\n"
    "pop %ebx\n"
    "ret\n"
#else
#error Unsupported target platform
#endif
    ".popsection\n"
    );

void ReturnFromCloneSyscall(SecureMem::Args *secureMem) {
  // TODO(mseaborn): Call sigreturn() to unblock signals.
#if defined(__x86_64__)
  // Get stack argument that was passed to clone().
  void **stack = (void**) secureMem->rsi;
  // Account for the x86-64 red zone adjustment that our syscall
  // interceptor does when returning from the system call.
  stack = (void**) ((char*) stack - 128);
  *--stack = secureMem->ret;
  *--stack = secureMem->r15;
  *--stack = secureMem->r14;
  *--stack = secureMem->r13;
  *--stack = secureMem->r12;
  *--stack = secureMem->r11;
  *--stack = secureMem->r10;
  *--stack = secureMem->r9;
  *--stack = secureMem->r8;
  *--stack = secureMem->rdi;
  *--stack = secureMem->rsi;
  *--stack = secureMem->rdx;
  *--stack = secureMem->rcx;
  *--stack = secureMem->rbx;
  *--stack = secureMem->rbp;
  asm("mov %0, %%rsp\n"
      "pop %%rbp\n"
      "pop %%rbx\n"
      "pop %%rcx\n"
      "pop %%rdx\n"
      "pop %%rsi\n"
      "pop %%rdi\n"
      "pop %%r8\n"
      "pop %%r9\n"
      "pop %%r10\n"
      "pop %%r11\n"
      "pop %%r12\n"
      "pop %%r13\n"
      "pop %%r14\n"
      "pop %%r15\n"
      "mov $0, %%rax\n"
      "ret\n"
      : : "m"(stack));
#elif defined(__i386__)
  // Get stack argument that was passed to clone().
  void **stack = (void**) secureMem->ecx;
  *--stack = secureMem->ret;
  *--stack = secureMem->ebp;
  *--stack = secureMem->edi;
  *--stack = secureMem->esi;
  *--stack = secureMem->edx;
  *--stack = secureMem->ecx;
  *--stack = secureMem->ebx;
  asm("mov %0, %%esp\n"
      "pop %%ebx\n"
      "pop %%ecx\n"
      "pop %%edx\n"
      "pop %%esi\n"
      "pop %%edi\n"
      "pop %%ebp\n"
      "mov $0, %%eax\n"
      "ret\n"
      : : "m"(stack));
#else
#error Unsupported target platform
#endif
}

void InitCustomTLS(void *addr) {
  Sandbox::SysCalls sys;
#if defined(__x86_64__)
  int rc = sys.arch_prctl(ARCH_SET_GS, addr);
  assert(rc == 0);
#elif defined(__i386__)
  struct user_desc u;
  u.entry_number    = (typeof u.entry_number)-1;
  u.base_addr       = (long) addr;
  u.limit           = 0xfffff;
  u.seg_32bit       = 1;
  u.contents        = 0;
  u.read_exec_only  = 0;
  u.limit_in_pages  = 1;
  u.seg_not_present = 0;
  u.useable         = 1;
  int rc = sys.set_thread_area(&u);
  assert(rc == 0);
  asm volatile("movw %w0, %%fs"
               : : "q"(8 * u.entry_number + 3));
#else
#error Unsupported target platform
#endif
}

void UnlockSyscallMutex() {
  Sandbox::SysCalls sys;
  // TODO(mseaborn): Use clone() to be the same as trusted_thread.cc.
  int pid = sys.fork();
  assert(pid >= 0);
  if (pid == 0) {
    int rc = sys.mprotect(&Sandbox::syscall_mutex_, 0x1000,
                          PROT_READ | PROT_WRITE);
    assert(rc == 0);
    Mutex::unlockMutex(&Sandbox::syscall_mutex_);
    sys._exit(0);
  }
  int status;
  int rc = sys.waitpid(pid, &status, 0);
  assert(rc == pid);
  assert(status == 0);
}

// Allocate a stack that is never freed.
char *AllocateStack() {
  Sandbox::SysCalls sys;
  int stack_size = 0x1000;
#if defined(__i386__)
  void *stack = sys.mmap2(NULL, stack_size, PROT_READ | PROT_WRITE,
                          MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
#else
  void *stack = sys.mmap(NULL, stack_size, PROT_READ | PROT_WRITE,
                         MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
#endif
  assert(stack != MAP_FAILED);
  return (char *) stack + stack_size;
}

int HandleNewThread(void *arg) {
  SecureMem::Args *secureMem = (SecureMem::Args *) arg;
  CreateReferenceTrustedThread(secureMem->newSecureMem);
  ReturnFromCloneSyscall(secureMem);
  return 0;
}

struct ThreadArgs {
  SecureMem::Args *secureMem;
  int threadFd;
};

int TrustedThread(void *arg) {
  struct ThreadArgs *args = (struct ThreadArgs *) arg;
  SecureMem::Args *secureMem = args->secureMem;
  int fd = args->threadFd;
  Sandbox::SysCalls sys;

  int sequence_no = 2;
  while (1) {
    long syscall_args[7];
    memset(syscall_args, 0, sizeof(syscall_args));
    int got = sys.read(fd, syscall_args, 4);
    assert(got == 4);

    long syscall_result;
    int sysnum = syscall_args[0];
    if (sysnum == -1 || sysnum == -2) {
      // Syscall where the registers have been checked by the trusted
      // process, e.g. munmap() ranges must be OK.
      // Doesn't need extra memory region.
      assert(secureMem->sequence == sequence_no);
      sequence_no += 2;
      if (secureMem->syscallNum == __NR_exit) {
        int rc = sys.close(fd);
        assert(rc == 0);
        rc = sys.close(secureMem->threadFdPub);
        assert(rc == 0);
      }
      else if (secureMem->syscallNum == __NR_clone) {
        assert(sysnum == -2);
        // Note that HandleNewThread() does UnlockSyscallMutex() for us.
#if defined(__x86_64__)
        long clone_flags = (long) secureMem->rdi;
        int *pid_ptr = (int *) secureMem->rdx;
        int *tid_ptr = (int *) secureMem->r10;
        void *tls_info = secureMem->r8;
#elif defined(__i386__)
        long clone_flags = (long) secureMem->ebx;
        int *pid_ptr = (int *) secureMem->edx;
        void *tls_info = secureMem->esi;
        int *tid_ptr = (int *) secureMem->edi;
#else
#error Unsupported target platform
#endif
        syscall_result = sys.clone(HandleNewThread, (void *) AllocateStack(),
                                   clone_flags, (void *) secureMem,
                                   pid_ptr, tls_info, tid_ptr);
        assert(syscall_result > 0);
        int sent = sys.write(fd, &syscall_result, sizeof(syscall_result));
        assert(sent == sizeof(syscall_result));
        continue;
      }
      memcpy(syscall_args, &secureMem->syscallNum, sizeof(syscall_args));
      // We release the mutex before doing the syscall, because we
      // have now copied the arguments.
      if (sysnum == -2)
        UnlockSyscallMutex();
    }
    else if (sysnum == -3) {
      // RDTSC request.  Send back a dummy answer.
      int timestamp[2] = {0, 0};
      int sent = sys.write(fd, (char *) timestamp, sizeof(timestamp));
      assert(sent == 8);
      continue;
    }
    else {
      int rest_size = sizeof(syscall_args) - sizeof(long);
      got = sys.read(fd, &syscall_args[1], rest_size);
      assert(got == rest_size);
    }
    syscall_result = DoSyscall(syscall_args);
    int sent = sys.write(fd, &syscall_result, sizeof(syscall_result));
    assert(sent == sizeof(syscall_result));
  }
  return 0;
}

void CreateReferenceTrustedThread(SecureMem::Args *secureMem) {
  // We are in the nascent thread.
  Sandbox::SysCalls sys;

  int socks[2];
  int rc = sys.socketpair(AF_UNIX, SOCK_STREAM, 0, socks);
  assert(rc == 0);
  int threadFdPub = socks[0];
  int threadFd = socks[1];

  // Create trusted thread.
  // We omit CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID.
  int flags = CLONE_VM | CLONE_FS | CLONE_FILES |
    CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM;
  // Assumes that the stack grows down.
  char *stack_top = AllocateStack() - sizeof(struct ThreadArgs);
  struct ThreadArgs *thread_args = (struct ThreadArgs *) stack_top;
  thread_args->threadFd = threadFd;
  thread_args->secureMem = secureMem;
  rc = sys.clone(TrustedThread, stack_top, flags, thread_args,
                 NULL, NULL, NULL);
  assert(rc > 0);

  // Make the thread state pages usable.
  rc = sys.mprotect(secureMem, 0x1000, PROT_READ);
  assert(rc == 0);
  rc = sys.mprotect(((char *) secureMem) + 0x1000, 0x1000,
                    PROT_READ | PROT_WRITE);
  assert(rc == 0);

  // Using cookie as the start is a little odd because other code in
  // syscall.c works around this when addressing from %fs.
  void *tls = (void*) &secureMem->cookie;
  InitCustomTLS(tls);

  UnlockSyscallMutex();

  int tid = sys.gettid();
  assert(tid > 0);

  // Send the socket FDs to the trusted process.
  // TODO(mseaborn): Don't duplicate this struct in trusted_process.cc
  struct {
    SecureMem::Args* self;
    int              tid;
    int              fdPub;
  } __attribute__((packed)) data;
  data.self = secureMem;
  data.tid = tid;
  data.fdPub = threadFdPub;
  bool ok = Sandbox::sendFd(Sandbox::cloneFdPub_, threadFdPub, threadFd,
                            (void *) &data, sizeof(data));
  assert(ok);

  // Wait for the trusted process to fill out the thread state for us.
  char byte_message;
  int got = sys.read(threadFdPub, &byte_message, 1);
  assert(got == 1);
  assert(byte_message == 0);

  // Switch to seccomp mode.
  rc = sys.prctl(PR_SET_SECCOMP, 1);
  assert(rc == 0);
}

}
<|endoftext|>


<commit_before>/*
-----------------------------------------------------------------------------
Copyright (c) 2010 Tippett Studio
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

1. Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the distribution.

3. The name of the author may not be used to endorse or promote products
   derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-----------------------------------------------------------------------------
*/

#include <iostream>
#include <vector>

#include <SiteShotgun/SiteProject.h>

namespace SiteSG {

// *****************************************************************************
SiteProject::SiteProject(SG::Shotgun *sg, const xmlrpc_c::value &attrs)
    : SG::Project(sg, attrs)
{
    m_classType = "SiteProject";
}

// *****************************************************************************
SiteProject::SiteProject(const SiteProject &ref)
    : Project(ref.m_sg, *ref.m_attrs)
{
    m_classType = "SiteProject";
}

// *****************************************************************************
SiteProject::~SiteProject()
{
    // Nothing
}

// *****************************************************************************
SG::List SiteProject::defaultReturnFields()
{
    return Project::defaultReturnFields()
               .append("created_by");
}

} // End namespace SiteSG
<commit_msg>* Minor update on the namespace.<commit_after>/*
-----------------------------------------------------------------------------
Copyright (c) 2010 Tippett Studio
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

1. Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the distribution.

3. The name of the author may not be used to endorse or promote products
   derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-----------------------------------------------------------------------------
*/

#include <iostream>
#include <vector>

#include <SiteShotgun/SiteProject.h>

namespace SiteSG {

// *****************************************************************************
SiteProject::SiteProject(SG::Shotgun *sg, const xmlrpc_c::value &attrs)
    : SG::Project(sg, attrs)
{
    m_classType = "SiteProject";
}

// *****************************************************************************
SiteProject::SiteProject(const SiteProject &ref)
    : SG::Project(ref.m_sg, *ref.m_attrs)
{
    m_classType = "SiteProject";
}

// *****************************************************************************
SiteProject::~SiteProject()
{
    // Nothing
}

// *****************************************************************************
SG::List SiteProject::defaultReturnFields()
{
    return SG::Project::defaultReturnFields()
               .append("created_by");
}

} // End namespace SiteSG
<|endoftext|>