<commit_before>/*****************************************************************************
 * Licensed to Qualys, Inc. (QUALYS) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * QUALYS licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ****************************************************************************/

/**
 * @file
 * @brief IronBee --- ParserSuite Command Line Tool
 *
 * Simple command line tool to feed input through ParserSuite.
 *
 * @author Christopher Alfeld <calfeld@qualys.com>
 */

#include "parser_suite.hpp"

#include <boost/bind.hpp>
#include <boost/bind/protect.hpp>
#include <boost/chrono.hpp>
#include <boost/function.hpp>
#include <boost/scoped_array.hpp>
#include <boost/type_traits.hpp>

#include <fstream>

using namespace std;
using namespace IronBee::ParserSuite;

typedef boost::function<void(ostream&, span_t&, uint64_t&)> parser_t;

template <typename R>
void simple_parser_func(
    boost::function<R(span_t&)> f,
    ostream&                    o,
    span_t&                     input,
    uint64_t&                   elapsed
)
{
    using namespace boost::chrono;

    time_point<high_resolution_clock> start;
    time_point<high_resolution_clock> end;
    start = high_resolution_clock::now();
    R result = f(input);
    end = high_resolution_clock::now();

    elapsed += duration_cast<microseconds>(end-start).count();
    o <<  result;
}

template <typename F>
parser_t simple_parser(F f)
{
    // Note use of protect.  Otherwise, bind will have type errors as it
    // fails to disambiguate the outer _1 from the inner _1.
    return boost::bind(
        simple_parser_func<typename F::result_type>,
        boost::protect(f),
        _1, _2, _3
    );
};

template <typename R>
parser_t simple_parser(R (*f)(span_t&))
{
    return simple_parser(boost::function<R(span_t&)>(f));
}

void read_all(istream& in, vector<char>& data)
{
    static const size_t c_buffer_size = 1024;
    data.clear();
    while (in) {
        size_t pre_size = data.size();
        data.resize(pre_size + c_buffer_size);
        in.read(data.data() + pre_size, c_buffer_size);
        data.resize(pre_size + in.gcount());
    }
}

int main(int argc, char **argv)
{
    typedef map<string, parser_t> parsers_t;
    parsers_t parsers;

    parsers["uri"]           = simple_parser(parse_uri);
    parsers["request_line"]  = simple_parser(parse_request_line);
    parsers["response_line"] = simple_parser(parse_response_line);
    parsers["headers"]       = simple_parser(parse_headers);
    parsers["request"]       = simple_parser(parse_request);
    parsers["response"]      = simple_parser(parse_response);
    parsers["authority"]     = simple_parser(parse_authority);
    parsers["path"] =
        simple_parser(boost::bind(parse_path, _1, '/', '.'));

    if (argc != 2) {
        cerr << "Usage: " << argv[0] << " <parser>" << endl;
        cerr << "Submit input on stdin." << endl;
        return 1;
    }

    parsers_t::const_iterator i = parsers.find(argv[1]);
    if (i == parsers.end()) {
        cerr << "No such parser: " << argv[1] << endl;
        return 1;
    }

    vector<char> raw_data;
    read_all(cin, raw_data);
    span_t input(raw_data.data(), raw_data.data() + raw_data.size());

    const uint64_t total_bytes = input.end() - input.begin();
    uint64_t total_elapsed = 0;
    uint64_t num_runs = 0;
    const char* old_begin = input.begin();
    while (! input.empty()) {
        uint64_t elapsed = 0;
        try {
            i->second(cout, input, elapsed);
        }
        catch (const IronBee::ParserSuite::error& e) {
            cout << "Error: " << endl;
            cout << diagnostic_information(e);
            return 1;
        }
        if (input.begin() == old_begin) {
            cout << "Error: No progress made." << endl;
            return 1;
        }

        ++num_runs;
        total_elapsed += elapsed;
        cout << "elapsed: " << elapsed << " us" << endl;
        cout << "consumed: " << input.begin() - old_begin << " bytes" << endl;
        old_begin = input.begin();
    }
    cout << "total_elapsed: " << total_elapsed << " us" << endl;
    if (num_runs > 0) {
        cout << "mean_elapsed: " << total_elapsed / num_runs << " us" << endl;
    }
    cout << "rate: " << total_bytes / (double(total_elapsed) / 1e6) << " bps" << endl;
    return 0;
}
<commit_msg>modules/psparse: Improve error handling. (CID 1062892)<commit_after>/*****************************************************************************
 * Licensed to Qualys, Inc. (QUALYS) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * QUALYS licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ****************************************************************************/

/**
 * @file
 * @brief IronBee --- ParserSuite Command Line Tool
 *
 * Simple command line tool to feed input through ParserSuite.
 *
 * @author Christopher Alfeld <calfeld@qualys.com>
 */

#include "parser_suite.hpp"

#include <boost/bind.hpp>
#include <boost/bind/protect.hpp>
#include <boost/chrono.hpp>
#include <boost/function.hpp>
#include <boost/scoped_array.hpp>
#include <boost/type_traits.hpp>

#include <fstream>

using namespace std;
using namespace IronBee::ParserSuite;

typedef boost::function<void(ostream&, span_t&, uint64_t&)> parser_t;

template <typename R>
void simple_parser_func(
    boost::function<R(span_t&)> f,
    ostream&                    o,
    span_t&                     input,
    uint64_t&                   elapsed
)
{
    using namespace boost::chrono;

    time_point<high_resolution_clock> start;
    time_point<high_resolution_clock> end;
    start = high_resolution_clock::now();
    R result = f(input);
    end = high_resolution_clock::now();

    elapsed += duration_cast<microseconds>(end-start).count();
    o <<  result;
}

template <typename F>
parser_t simple_parser(F f)
{
    // Note use of protect.  Otherwise, bind will have type errors as it
    // fails to disambiguate the outer _1 from the inner _1.
    return boost::bind(
        simple_parser_func<typename F::result_type>,
        boost::protect(f),
        _1, _2, _3
    );
};

template <typename R>
parser_t simple_parser(R (*f)(span_t&))
{
    return simple_parser(boost::function<R(span_t&)>(f));
}

void read_all(istream& in, vector<char>& data)
{
    static const size_t c_buffer_size = 1024;
    data.clear();
    while (in) {
        size_t pre_size = data.size();
        data.resize(pre_size + c_buffer_size);
        in.read(data.data() + pre_size, c_buffer_size);
        data.resize(pre_size + in.gcount());
    }
}

int main(int argc, char **argv)
{
    typedef map<string, parser_t> parsers_t;
    parsers_t parsers;

    parsers["uri"]           = simple_parser(parse_uri);
    parsers["request_line"]  = simple_parser(parse_request_line);
    parsers["response_line"] = simple_parser(parse_response_line);
    parsers["headers"]       = simple_parser(parse_headers);
    parsers["request"]       = simple_parser(parse_request);
    parsers["response"]      = simple_parser(parse_response);
    parsers["authority"]     = simple_parser(parse_authority);
    parsers["path"] =
        simple_parser(boost::bind(parse_path, _1, '/', '.'));

    if (argc != 2) {
        cerr << "Usage: " << argv[0] << " <parser>" << endl;
        cerr << "Submit input on stdin." << endl;
        return 1;
    }

    parsers_t::const_iterator i = parsers.find(argv[1]);
    if (i == parsers.end()) {
        cerr << "No such parser: " << argv[1] << endl;
        return 1;
    }

    vector<char> raw_data;
    read_all(cin, raw_data);
    span_t input(raw_data.data(), raw_data.data() + raw_data.size());

    const uint64_t total_bytes = input.end() - input.begin();
    uint64_t total_elapsed = 0;
    uint64_t num_runs = 0;
    const char* old_begin = input.begin();
    while (! input.empty()) {
        uint64_t elapsed = 0;
        try {
            i->second(cout, input, elapsed);
        }
        catch (const boost::exception& e) {
            cout << "Error: " << endl;
            cout << diagnostic_information(e);
            return 1;
        }
        catch (const exception& e) {
            cout << "Error: " << e.what() << endl;
            return 1;
        }
        if (input.begin() == old_begin) {
            cout << "Error: No progress made." << endl;
            return 1;
        }

        ++num_runs;
        total_elapsed += elapsed;
        cout << "elapsed: " << elapsed << " us" << endl;
        cout << "consumed: " << input.begin() - old_begin << " bytes" << endl;
        old_begin = input.begin();
    }
    cout << "total_elapsed: " << total_elapsed << " us" << endl;
    if (num_runs > 0) {
        cout << "mean_elapsed: " << total_elapsed / num_runs << " us" << endl;
    }
    cout << "rate: " << total_bytes / (double(total_elapsed) / 1e6) << " bps" << endl;
    return 0;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/experimental/JemallocHugePageAllocator.h>

#include <sstream>

#include <folly/portability/Malloc.h>
#include <folly/portability/String.h>
#include <folly/portability/SysMman.h>

#include <glog/logging.h>

#if (defined(MADV_HUGEPAGE) || defined(MAP_ALIGNED_SUPER)) && \
    defined(FOLLY_USE_JEMALLOC) && !FOLLY_SANITIZE
#if defined(__FreeBSD__) || (JEMALLOC_VERSION_MAJOR >= 5)
#define FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED 1
bool folly::JemallocHugePageAllocator::hugePagesSupported{true};
#endif

#endif // MADV_HUGEPAGE || MAP_ALIGNED_SUPER && defined(FOLLY_USE_JEMALLOC) &&
       // !FOLLY_SANITIZE

#ifndef FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED
// Some mocks when jemalloc.h is not included or version too old
// or when the system does not support the MADV_HUGEPAGE madvise flag
#undef MALLOCX_ARENA
#undef MALLOCX_TCACHE_NONE
#undef MADV_HUGEPAGE
#define MALLOCX_ARENA(x) 0
#define MALLOCX_TCACHE_NONE 0
#define MADV_HUGEPAGE 0

#if !defined(JEMALLOC_VERSION_MAJOR) || (JEMALLOC_VERSION_MAJOR < 5)
typedef struct extent_hooks_s extent_hooks_t;
typedef void*(
    extent_alloc_t)(extent_hooks_t*, void*, size_t, size_t, bool*, bool*, unsigned);
struct extent_hooks_s {
  extent_alloc_t* alloc;
};
#endif // JEMALLOC_VERSION_MAJOR

bool folly::JemallocHugePageAllocator::hugePagesSupported{false};

#endif // FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED

namespace folly {
namespace {

void print_error(int err, const char* msg) {
  int cur_errno = std::exchange(errno, err);
  PLOG(ERROR) << msg;
  errno = cur_errno;
}

class HugePageArena {
 public:
  int init(int initial_nr_pages, int max_nr_pages);

  /* forces the system to actually back more pages */
  void init_more(int nr_pages);

  void* reserve(size_t size, size_t alignment);

  bool addressInArena(void* address) {
    auto addr = reinterpret_cast<uintptr_t>(address);
    return addr >= start_ && addr < protEnd_;
  }

  size_t freeSpace() { return end_ - freePtr_; }

  unsigned arenaIndex() { return arenaIndex_; }

 private:
  void map_pages(size_t initial_nr_pages, size_t max_nr_pages);

  bool setup_next_pages(uintptr_t nextFreePtr);

  static void* allocHook(
      extent_hooks_t* extent,
      void* new_addr,
      size_t size,
      size_t alignment,
      bool* zero,
      bool* commit,
      unsigned arena_ind);

  uintptr_t start_{0};
  uintptr_t end_{0};
  uintptr_t freePtr_{0};
  uintptr_t protEnd_{0};
  extent_alloc_t* originalAlloc_{nullptr};
  extent_hooks_t extentHooks_;
  unsigned arenaIndex_{0};
};

constexpr size_t kHugePageSize = 2 * 1024 * 1024;

// Singleton arena instance
HugePageArena arena;

// TODO: Always reserve a big chunk (16GB) of virtual address space up front?
// Doesn't allocate anything.
// const bool _inited = []() {
//   return folly::JemallocHugePageAllocator::init(0, 8192) != 0;
// }();

template <typename T, typename U>
static inline T align_up(T val, U alignment) {
  DCHECK((alignment & (alignment - 1)) == 0);
  return (val + alignment - 1) & ~(alignment - 1);
}

// mmap enough memory to hold the aligned huge pages, then use madvise
// to get huge pages. This can be checked in /proc/<pid>/smaps.
// If successful, sets the arena member pointers to reflect the mapped memory.
// Otherwise, leaves them unchanged (zeroed).
void HugePageArena::map_pages(size_t initial_nr_pages, size_t max_nr_pages) {
  // Initial mmapped area is large enough to contain the aligned huge pages
  size_t initial_alloc_size = initial_nr_pages * kHugePageSize;
  size_t max_alloc_size = max_nr_pages * kHugePageSize;
  int mflags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(__FreeBSD__)
  mflags |= MAP_ALIGNED_SUPER;
#endif
  void* p =
      mmap(nullptr, max_alloc_size + kHugePageSize, PROT_NONE, mflags, -1, 0);

  if (p == MAP_FAILED) {
    return;
  }

  // Aligned start address
  uintptr_t first_page = align_up((uintptr_t)p, kHugePageSize);

#if !defined(__FreeBSD__)
  // Unmap left-over 4k pages
  const size_t excess_head = first_page - (uintptr_t)p;
  const size_t excess_tail = kHugePageSize - excess_head;
  if (excess_head != 0) {
    munmap(p, excess_head);
  }
  if (excess_tail != 0) {
    munmap((void*)(first_page + max_alloc_size), excess_tail);
  }
#endif

  start_ = freePtr_ = protEnd_ = first_page;
  end_ = start_ + max_alloc_size;

  setup_next_pages(start_ + initial_alloc_size);
}

void HugePageArena::init_more(int nr_pages) {
  setup_next_pages(start_ + nr_pages * kHugePageSize);
}

// Warning: This can be called inside malloc(). Check the comments in
// HugePageArena::allocHook to understand the restrictions that imposes before
// making any change to this function.
// Requirement: upto > freePtr_ && upto > protEnd_.
// Returns whether the setup succeeded.
bool HugePageArena::setup_next_pages(uintptr_t upto) {
  const uintptr_t curPtr = protEnd_;
  const uintptr_t endPtr = align_up(upto, kHugePageSize);
  const size_t len = endPtr - curPtr;

  if (len == 0) {
    return true;
  }

  if (endPtr > end_) {
    return false;
  }

#if !defined(__FreeBSD__)
  // Tell the kernel to please give us huge pages for this range
  if (madvise((void*)curPtr, len, MADV_HUGEPAGE) != 0) {
    return false;
  }
#endif

  // Make this memory accessible.
  if (mprotect((void*)curPtr, len, PROT_READ | PROT_WRITE) != 0) {
    return false;
  }

#if !defined(__FreeBSD__)
  // With THP set to madvise, page faults on these pages will block until a
  // huge page is found to service it. However, if memory becomes fragmented
  // before these pages are touched, then we end up blocking for kcompactd to
  // make a page available. This increases pressure to the point that oomd comes
  // in and kill us :(. So, preemptively touch these pages to get them backed
  // as early as possible to prevent stalling due to no available huge pages.
  //
  // Note: this does not guarantee we won't be oomd killed here, it's just much
  // more unlikely given this should be among the very first things an
  // application does.
  for (uintptr_t ptr = curPtr; ptr < endPtr; ptr += kHugePageSize) {
    memset((void*)ptr, 0, 1);
  }
#endif

  protEnd_ = endPtr;
  return true;
}

// WARNING WARNING WARNING
// This function is the hook invoked on malloc path for the hugepage allocator.
// This means it should not, itself, call malloc. If any of the following
// happens within this function, it *WILL* cause a DEADLOCK (from the circular
// dependency):
// - any dynamic memory allocation (i.e. calls to malloc)
// - any operations that may lead to dynamic operations, such as logging (e.g.
//   LOG, VLOG, LOG_IF) and DCHECK.
// WARNING WARNING WARNING
void* HugePageArena::allocHook(
    extent_hooks_t* extent,
    void* new_addr,
    size_t size,
    size_t alignment,
    bool* zero,
    bool* commit,
    unsigned arena_ind) {
  assert((size & (kHugePageSize - 1)) == 0);
  void* res = nullptr;
  if (new_addr == nullptr) {
    res = arena.reserve(size, alignment);
  }
  if (res == nullptr) {
    res = arena.originalAlloc_(
        extent, new_addr, size, alignment, zero, commit, arena_ind);
  } else {
    if (*zero) {
      memset(res, 0, size);
    }
    *commit = true;
  }
  return res;
}

int HugePageArena::init(int initial_nr_pages, int max_nr_pages) {
  DCHECK(start_ == 0);
  DCHECK(usingJEMalloc());

  if (max_nr_pages < initial_nr_pages) {
    max_nr_pages = initial_nr_pages;
  }

  size_t len = sizeof(arenaIndex_);
  if (auto ret = mallctl("arenas.create", &arenaIndex_, &len, nullptr, 0)) {
    print_error(ret, "Unable to create arena");
    return 0;
  }

  // Set grow retained limit to stop jemalloc from
  // forever increasing the requested size after failed allocations.
  // Normally jemalloc asks for maps of increasing size in order to avoid
  // hitting the limit of allowed mmaps per process.
  // Since this arena is backed by a single mmap and is using huge pages,
  // this is not a concern here.
  // TODO: Support growth of the huge page arena.
  size_t mib[3];
  size_t miblen = sizeof(mib) / sizeof(size_t);
  std::ostringstream rtl_key;
  rtl_key << "arena." << arenaIndex_ << ".retain_grow_limit";
  if (auto ret = mallctlnametomib(rtl_key.str().c_str(), mib, &miblen)) {
    print_error(ret, "Unable to read growth limit");
    return 0;
  }
  size_t grow_retained_limit = kHugePageSize;
  mib[1] = arenaIndex_;
  if (auto ret = mallctlbymib(
          mib,
          miblen,
          nullptr,
          nullptr,
          &grow_retained_limit,
          sizeof(grow_retained_limit))) {
    print_error(ret, "Unable to set growth limit");
    return 0;
  }

  std::ostringstream hooks_key;
  hooks_key << "arena." << arenaIndex_ << ".extent_hooks";
  extent_hooks_t* hooks;
  len = sizeof(hooks);
  // Read the existing hooks
  if (auto ret = mallctl(hooks_key.str().c_str(), &hooks, &len, nullptr, 0)) {
    print_error(ret, "Unable to get the hooks");
    return 0;
  }
  originalAlloc_ = hooks->alloc;

  // Set the custom hook
  extentHooks_ = *hooks;
  extentHooks_.alloc = &allocHook;
  extent_hooks_t* new_hooks = &extentHooks_;
  if (auto ret = mallctl(
          hooks_key.str().c_str(),
          nullptr,
          nullptr,
          &new_hooks,
          sizeof(new_hooks))) {
    print_error(ret, "Unable to set the hooks");
    return 0;
  }

  // Set dirty decay and muzzy decay time to -1, which will cause jemalloc
  // to never free memory to kernel.
  ssize_t decay_ms = -1;
  std::ostringstream dirty_decay_key;
  dirty_decay_key << "arena." << arenaIndex_ << ".dirty_decay_ms";
  if (auto ret = mallctl(
          dirty_decay_key.str().c_str(),
          nullptr,
          nullptr,
          &decay_ms,
          sizeof(decay_ms))) {
    print_error(ret, "Unable to set dirty decay time");
    return 0;
  }
  std::ostringstream muzzy_decay_key;
  muzzy_decay_key << "arena." << arenaIndex_ << ".muzzy_decay_ms";
  if (auto ret = mallctl(
          muzzy_decay_key.str().c_str(),
          nullptr,
          nullptr,
          &decay_ms,
          sizeof(decay_ms))) {
    print_error(ret, "Unable to set muzzy decay time");
    return 0;
  }

  map_pages(initial_nr_pages, max_nr_pages);
  if (start_ == 0) {
    return false;
  }
  return MALLOCX_ARENA(arenaIndex_) | MALLOCX_TCACHE_NONE;
}

// Warning: Check the comments in HugePageArena::allocHook before making any
// change to this function.
void* HugePageArena::reserve(size_t size, size_t alignment) {
  uintptr_t res = align_up(freePtr_, alignment);
  uintptr_t newFreePtr = res + size;
  if (newFreePtr > end_) {
    return nullptr;
  }
  if (newFreePtr > protEnd_) {
    if (!setup_next_pages(newFreePtr)) {
      return nullptr;
    }
  }
  freePtr_ = newFreePtr;
  return reinterpret_cast<void*>(res);
}

} // namespace

int JemallocHugePageAllocator::flags_{0};

bool JemallocHugePageAllocator::init(int initial_nr_pages, int max_nr_pages) {
  if (!usingJEMalloc()) {
    LOG(ERROR) << "Not linked with jemalloc?";
    hugePagesSupported = false;
  }
  if (hugePagesSupported) {
    if (flags_ == 0) {
      flags_ = arena.init(initial_nr_pages, max_nr_pages);
    } else {
      /* was already initialized, let's just init the requested pages */
      arena.init_more(initial_nr_pages);
    }
  } else {
    LOG(WARNING) << "Huge Page Allocator not supported";
  }
  return flags_ != 0;
}

size_t JemallocHugePageAllocator::freeSpace() {
  return arena.freeSpace();
}

bool JemallocHugePageAllocator::addressInArena(void* address) {
  return arena.addressInArena(address);
}

unsigned arenaIndex() {
  return arena.arenaIndex();
}

} // namespace folly
<commit_msg>Remove Faulty Assert from JemallocHugePageAllocator<commit_after>/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/experimental/JemallocHugePageAllocator.h>

#include <sstream>

#include <folly/portability/Malloc.h>
#include <folly/portability/String.h>
#include <folly/portability/SysMman.h>

#include <glog/logging.h>

#if (defined(MADV_HUGEPAGE) || defined(MAP_ALIGNED_SUPER)) && \
    defined(FOLLY_USE_JEMALLOC) && !FOLLY_SANITIZE
#if defined(__FreeBSD__) || (JEMALLOC_VERSION_MAJOR >= 5)
#define FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED 1
bool folly::JemallocHugePageAllocator::hugePagesSupported{true};
#endif

#endif // MADV_HUGEPAGE || MAP_ALIGNED_SUPER && defined(FOLLY_USE_JEMALLOC) &&
       // !FOLLY_SANITIZE

#ifndef FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED
// Some mocks when jemalloc.h is not included or version too old
// or when the system does not support the MADV_HUGEPAGE madvise flag
#undef MALLOCX_ARENA
#undef MALLOCX_TCACHE_NONE
#undef MADV_HUGEPAGE
#define MALLOCX_ARENA(x) 0
#define MALLOCX_TCACHE_NONE 0
#define MADV_HUGEPAGE 0

#if !defined(JEMALLOC_VERSION_MAJOR) || (JEMALLOC_VERSION_MAJOR < 5)
typedef struct extent_hooks_s extent_hooks_t;
typedef void*(
    extent_alloc_t)(extent_hooks_t*, void*, size_t, size_t, bool*, bool*, unsigned);
struct extent_hooks_s {
  extent_alloc_t* alloc;
};
#endif // JEMALLOC_VERSION_MAJOR

bool folly::JemallocHugePageAllocator::hugePagesSupported{false};

#endif // FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED

namespace folly {
namespace {

void print_error(int err, const char* msg) {
  int cur_errno = std::exchange(errno, err);
  PLOG(ERROR) << msg;
  errno = cur_errno;
}

class HugePageArena {
 public:
  int init(int initial_nr_pages, int max_nr_pages);

  /* forces the system to actually back more pages */
  void init_more(int nr_pages);

  void* reserve(size_t size, size_t alignment);

  bool addressInArena(void* address) {
    auto addr = reinterpret_cast<uintptr_t>(address);
    return addr >= start_ && addr < protEnd_;
  }

  size_t freeSpace() { return end_ - freePtr_; }

  unsigned arenaIndex() { return arenaIndex_; }

 private:
  void map_pages(size_t initial_nr_pages, size_t max_nr_pages);

  bool setup_next_pages(uintptr_t nextFreePtr);

  static void* allocHook(
      extent_hooks_t* extent,
      void* new_addr,
      size_t size,
      size_t alignment,
      bool* zero,
      bool* commit,
      unsigned arena_ind);

  uintptr_t start_{0};
  uintptr_t end_{0};
  uintptr_t freePtr_{0};
  uintptr_t protEnd_{0};
  extent_alloc_t* originalAlloc_{nullptr};
  extent_hooks_t extentHooks_;
  unsigned arenaIndex_{0};
};

constexpr size_t kHugePageSize = 2 * 1024 * 1024;

// Singleton arena instance
HugePageArena arena;

// TODO: Always reserve a big chunk (16GB) of virtual address space up front?
// Doesn't allocate anything.
// const bool _inited = []() {
//   return folly::JemallocHugePageAllocator::init(0, 8192) != 0;
// }();

template <typename T, typename U>
static inline T align_up(T val, U alignment) {
  DCHECK((alignment & (alignment - 1)) == 0);
  return (val + alignment - 1) & ~(alignment - 1);
}

// mmap enough memory to hold the aligned huge pages, then use madvise
// to get huge pages. This can be checked in /proc/<pid>/smaps.
// If successful, sets the arena member pointers to reflect the mapped memory.
// Otherwise, leaves them unchanged (zeroed).
void HugePageArena::map_pages(size_t initial_nr_pages, size_t max_nr_pages) {
  // Initial mmapped area is large enough to contain the aligned huge pages
  size_t initial_alloc_size = initial_nr_pages * kHugePageSize;
  size_t max_alloc_size = max_nr_pages * kHugePageSize;
  int mflags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(__FreeBSD__)
  mflags |= MAP_ALIGNED_SUPER;
#endif
  void* p =
      mmap(nullptr, max_alloc_size + kHugePageSize, PROT_NONE, mflags, -1, 0);

  if (p == MAP_FAILED) {
    return;
  }

  // Aligned start address
  uintptr_t first_page = align_up((uintptr_t)p, kHugePageSize);

#if !defined(__FreeBSD__)
  // Unmap left-over 4k pages
  const size_t excess_head = first_page - (uintptr_t)p;
  const size_t excess_tail = kHugePageSize - excess_head;
  if (excess_head != 0) {
    munmap(p, excess_head);
  }
  if (excess_tail != 0) {
    munmap((void*)(first_page + max_alloc_size), excess_tail);
  }
#endif

  start_ = freePtr_ = protEnd_ = first_page;
  end_ = start_ + max_alloc_size;

  setup_next_pages(start_ + initial_alloc_size);
}

void HugePageArena::init_more(int nr_pages) {
  setup_next_pages(start_ + nr_pages * kHugePageSize);
}

// Warning: This can be called inside malloc(). Check the comments in
// HugePageArena::allocHook to understand the restrictions that imposes before
// making any change to this function.
// Requirement: upto > freePtr_ && upto > protEnd_.
// Returns whether the setup succeeded.
bool HugePageArena::setup_next_pages(uintptr_t upto) {
  const uintptr_t curPtr = protEnd_;
  const uintptr_t endPtr = align_up(upto, kHugePageSize);
  const size_t len = endPtr - curPtr;

  if (len == 0) {
    return true;
  }

  if (endPtr > end_) {
    return false;
  }

#if !defined(__FreeBSD__)
  // Tell the kernel to please give us huge pages for this range
  if (madvise((void*)curPtr, len, MADV_HUGEPAGE) != 0) {
    return false;
  }
#endif

  // Make this memory accessible.
  if (mprotect((void*)curPtr, len, PROT_READ | PROT_WRITE) != 0) {
    return false;
  }

#if !defined(__FreeBSD__)
  // With THP set to madvise, page faults on these pages will block until a
  // huge page is found to service it. However, if memory becomes fragmented
  // before these pages are touched, then we end up blocking for kcompactd to
  // make a page available. This increases pressure to the point that oomd comes
  // in and kill us :(. So, preemptively touch these pages to get them backed
  // as early as possible to prevent stalling due to no available huge pages.
  //
  // Note: this does not guarantee we won't be oomd killed here, it's just much
  // more unlikely given this should be among the very first things an
  // application does.
  for (uintptr_t ptr = curPtr; ptr < endPtr; ptr += kHugePageSize) {
    memset((void*)ptr, 0, 1);
  }
#endif

  protEnd_ = endPtr;
  return true;
}

// WARNING WARNING WARNING
// This function is the hook invoked on malloc path for the hugepage allocator.
// This means it should not, itself, call malloc. If any of the following
// happens within this function, it *WILL* cause a DEADLOCK (from the circular
// dependency):
// - any dynamic memory allocation (i.e. calls to malloc)
// - any operations that may lead to dynamic operations, such as logging (e.g.
//   LOG, VLOG, LOG_IF) and DCHECK.
// WARNING WARNING WARNING
void* HugePageArena::allocHook(
    extent_hooks_t* extent,
    void* new_addr,
    size_t size,
    size_t alignment,
    bool* zero,
    bool* commit,
    unsigned arena_ind) {
  void* res = nullptr;
  if (new_addr == nullptr) {
    res = arena.reserve(size, alignment);
  }
  if (res == nullptr) {
    res = arena.originalAlloc_(
        extent, new_addr, size, alignment, zero, commit, arena_ind);
  } else {
    if (*zero) {
      memset(res, 0, size);
    }
    *commit = true;
  }
  return res;
}

int HugePageArena::init(int initial_nr_pages, int max_nr_pages) {
  DCHECK(start_ == 0);
  DCHECK(usingJEMalloc());

  if (max_nr_pages < initial_nr_pages) {
    max_nr_pages = initial_nr_pages;
  }

  size_t len = sizeof(arenaIndex_);
  if (auto ret = mallctl("arenas.create", &arenaIndex_, &len, nullptr, 0)) {
    print_error(ret, "Unable to create arena");
    return 0;
  }

  // Set grow retained limit to stop jemalloc from
  // forever increasing the requested size after failed allocations.
  // Normally jemalloc asks for maps of increasing size in order to avoid
  // hitting the limit of allowed mmaps per process.
  // Since this arena is backed by a single mmap and is using huge pages,
  // this is not a concern here.
  // TODO: Support growth of the huge page arena.
  size_t mib[3];
  size_t miblen = sizeof(mib) / sizeof(size_t);
  std::ostringstream rtl_key;
  rtl_key << "arena." << arenaIndex_ << ".retain_grow_limit";
  if (auto ret = mallctlnametomib(rtl_key.str().c_str(), mib, &miblen)) {
    print_error(ret, "Unable to read growth limit");
    return 0;
  }
  size_t grow_retained_limit = kHugePageSize;
  mib[1] = arenaIndex_;
  if (auto ret = mallctlbymib(
          mib,
          miblen,
          nullptr,
          nullptr,
          &grow_retained_limit,
          sizeof(grow_retained_limit))) {
    print_error(ret, "Unable to set growth limit");
    return 0;
  }

  std::ostringstream hooks_key;
  hooks_key << "arena." << arenaIndex_ << ".extent_hooks";
  extent_hooks_t* hooks;
  len = sizeof(hooks);
  // Read the existing hooks
  if (auto ret = mallctl(hooks_key.str().c_str(), &hooks, &len, nullptr, 0)) {
    print_error(ret, "Unable to get the hooks");
    return 0;
  }
  originalAlloc_ = hooks->alloc;

  // Set the custom hook
  extentHooks_ = *hooks;
  extentHooks_.alloc = &allocHook;
  extent_hooks_t* new_hooks = &extentHooks_;
  if (auto ret = mallctl(
          hooks_key.str().c_str(),
          nullptr,
          nullptr,
          &new_hooks,
          sizeof(new_hooks))) {
    print_error(ret, "Unable to set the hooks");
    return 0;
  }

  // Set dirty decay and muzzy decay time to -1, which will cause jemalloc
  // to never free memory to kernel.
  ssize_t decay_ms = -1;
  std::ostringstream dirty_decay_key;
  dirty_decay_key << "arena." << arenaIndex_ << ".dirty_decay_ms";
  if (auto ret = mallctl(
          dirty_decay_key.str().c_str(),
          nullptr,
          nullptr,
          &decay_ms,
          sizeof(decay_ms))) {
    print_error(ret, "Unable to set dirty decay time");
    return 0;
  }
  std::ostringstream muzzy_decay_key;
  muzzy_decay_key << "arena." << arenaIndex_ << ".muzzy_decay_ms";
  if (auto ret = mallctl(
          muzzy_decay_key.str().c_str(),
          nullptr,
          nullptr,
          &decay_ms,
          sizeof(decay_ms))) {
    print_error(ret, "Unable to set muzzy decay time");
    return 0;
  }

  map_pages(initial_nr_pages, max_nr_pages);
  if (start_ == 0) {
    return false;
  }
  return MALLOCX_ARENA(arenaIndex_) | MALLOCX_TCACHE_NONE;
}

// Warning: Check the comments in HugePageArena::allocHook before making any
// change to this function.
void* HugePageArena::reserve(size_t size, size_t alignment) {
  uintptr_t res = align_up(freePtr_, alignment);
  uintptr_t newFreePtr = res + size;
  if (newFreePtr > end_) {
    return nullptr;
  }
  if (newFreePtr > protEnd_) {
    if (!setup_next_pages(newFreePtr)) {
      return nullptr;
    }
  }
  freePtr_ = newFreePtr;
  return reinterpret_cast<void*>(res);
}

} // namespace

int JemallocHugePageAllocator::flags_{0};

bool JemallocHugePageAllocator::init(int initial_nr_pages, int max_nr_pages) {
  if (!usingJEMalloc()) {
    LOG(ERROR) << "Not linked with jemalloc?";
    hugePagesSupported = false;
  }
  if (hugePagesSupported) {
    if (flags_ == 0) {
      flags_ = arena.init(initial_nr_pages, max_nr_pages);
    } else {
      /* was already initialized, let's just init the requested pages */
      arena.init_more(initial_nr_pages);
    }
  } else {
    LOG(WARNING) << "Huge Page Allocator not supported";
  }
  return flags_ != 0;
}

size_t JemallocHugePageAllocator::freeSpace() {
  return arena.freeSpace();
}

bool JemallocHugePageAllocator::addressInArena(void* address) {
  return arena.addressInArena(address);
}

unsigned arenaIndex() {
  return arena.arenaIndex();
}

} // namespace folly
<|endoftext|>


<commit_before>namespace gli
{
	namespace detail
	{
		inline image::mipmap duplicate(image::mipmap const & Mipmap2D)
		{
			image::mipmap Result(Mipmap2D.dimensions(), Mipmap2D.format());
			memcpy(Result.data(), Mipmap2D.data(), Mipmap2D.capacity());
			return Result;
		}

		inline image::mipmap flip(image::mipmap const & Mipmap2D)
		{
			image::mipmap Result(Mipmap2D.dimensions(), Mipmap2D.format());

			std::size_t ValueSize = Result.value_size();
			glm::byte * DstPtr = Result.data();
			glm::byte const * const SrcPtr = Mipmap2D.data();

			for(std::size_t j = 0; j < Result.dimensions().y; ++j)
			for(std::size_t i = 0; i < Result.dimensions().x; ++i)
			{
				std::size_t DstIndex = (i + j * Result.dimensions().y) * ValueSize;
				std::size_t SrcIndex = (i + (Result.dimensions().y - j) * Result.dimensions().x) * ValueSize;
				memcpy(DstPtr + DstIndex, SrcPtr + SrcIndex, ValueSize);
			}

			return Result;
		}

		inline image::mipmap mirror(image::mipmap const & Mipmap2D)
		{
			image::mipmap Result(Mipmap2D.dimensions(), Mipmap2D.format());

			std::size_t ValueSize = Mipmap2D.value_size();
			glm::byte * DstPtr = Result.data();
			glm::byte const * const SrcPtr = Mipmap2D.data();

			for(std::size_t j = 0; j < Result.dimensions().y; ++j)
			for(std::size_t i = 0; i < Result.dimensions().x; ++i)
			{
				std::size_t DstIndex = (i + j * Result.dimensions().x) * ValueSize;
				std::size_t SrcIndex = ((Result.dimensions().x - i) + j * Result.dimensions().x) * ValueSize;
				memcpy(DstPtr + DstIndex, SrcPtr + SrcIndex, ValueSize);
			}

			return Result;
		}

		inline image::mipmap swizzle
		(
			image::mipmap const & Mipmap,
			glm::uvec4 const & Channel
		)
		{
			image::mipmap Result = detail::duplicate(Mipmap);

			glm::byte * DataDst = Result.data();
			glm::byte const * const DataSrc = Mipmap.data();

			gli::image::size_type CompSize = Mipmap.value_size() / Mipmap.components();
			gli::image::size_type TexelCount = Mipmap.capacity() / Mipmap.value_size();

			for(gli::image::size_type t = 0; t < TexelCount; ++t)
			for(gli::image::size_type c = 0; c < Mipmap.components(); ++c)
			{
				gli::image::size_type IndexSrc = t * Mipmap.components() + Channel[glm::uvec4::size_type(c)];
				gli::image::size_type IndexDst = t * Mipmap.components() + c;

				memcpy(DataDst + IndexDst, DataSrc + IndexSrc, CompSize);
			}

			return Result;
		}

		inline image::mipmap crop
		(
			image::mipmap const & Mipmap,
			glm::uvec2 const & Position,
			glm::uvec2 const & Size
		)
		{
			assert((Position.x + Size.x) <= Mipmap.dimensions().x && (Position.y + Size.y) <= Mipmap.dimensions().y);

			image::mipmap Result(glm::uvec3(Size, glm::uint(1)), Mipmap.format());

			glm::byte* DstData = Result.data();
			glm::byte const * const SrcData = Mipmap.data();

			for(std::size_t j = 0; j < Size.y; ++j)
			{
				std::size_t DstIndex = 0                                + (0          + j) * Size.x         * Mipmap.value_size();
				std::size_t SrcIndex = Position.x * Mipmap.value_size() + (Position.y + j) * Mipmap.dimensions().x * Mipmap.value_size();
				memcpy(DstData + DstIndex, SrcData + SrcIndex, Mipmap.value_size() * Size.x);
			}

			return Result;
		}

		inline image::mipmap copy
		(
			image::mipmap const & SrcMipmap,
			glm::uvec2 const & SrcPosition,
			glm::uvec2 const & SrcSize,
			image::mipmap & DstMipmap,
			glm::uvec2 const & DstPosition
		)
		{
			assert((SrcPosition.x + SrcSize.x) <= SrcMipmap.dimensions().x && (SrcPosition.y + SrcSize.y) <= SrcMipmap.dimensions().y);
			assert(SrcMipmap.format() == DstMipmap.format());

			glm::byte * DstData = DstMipmap.data();
			glm::byte const * const SrcData = SrcMipmap.data();

			std::size_t SizeX = std::min(std::size_t(SrcSize.x + SrcPosition.x), std::size_t(DstMipmap.dimensions().x  + DstPosition.x));
			std::size_t SizeY = std::min(std::size_t(SrcSize.y + SrcPosition.y), std::size_t(DstMipmap.dimensions().y + DstPosition.y));

			for(std::size_t j = 0; j < SizeY; ++j)
			{
				std::size_t DstIndex = DstPosition.x * DstMipmap.value_size() + (DstPosition.y + j) * DstMipmap.dimensions().x * DstMipmap.value_size();
				std::size_t SrcIndex = SrcPosition.x * SrcMipmap.value_size() + (SrcPosition.y + j) * SrcMipmap.dimensions().x * SrcMipmap.value_size();
				memcpy(DstData + DstIndex, SrcData + SrcIndex, SrcMipmap.value_size() * SizeX);
			}

			return DstMipmap;
		}

	}//namespace detail

	inline image duplicate(image const & Image2D)
	{
		image Result(Image2D.levels());
		for(std::size_t Level = 0; Level < Image2D.levels(); ++Level)
			Result[Level] = detail::duplicate(Image2D[Level]);
		return Result;
	}

	inline image flip(image const & Image2D)
	{
		image Result(Image2D.levels());
		for(std::size_t Level = 0; Level < Image2D.levels(); ++Level)
			Result[Level] = detail::flip(Image2D[Level]);
		return Result;
	}

	inline image mirror(image const & Image2D)
	{
		image Result(Image2D.levels());
		for(std::size_t Level = 0; Level < Image2D.levels(); ++Level)
			Result[Level] = detail::mirror(Image2D[Level]);
		return Result;
	}

	inline image crop
	(
		image const & Image,
		glm::uvec2 const & Position,
		glm::uvec2 const & Size
	)
	{
		image Result(Image.levels());
		for(glm::uint Level = 0; Level < Image.levels(); ++Level)
			Result[Level] = detail::crop(Image[Level], Position >> Level, Size >> Level);
		return Result;
	}

	inline image swizzle
	(
		image const & Image,
		glm::uvec4 const & Channel
	)
	{
		image Result(Image.levels());
		for(std::size_t Level = 0; Level < Image.levels(); ++Level)
			Result[Level] = detail::swizzle(Image[Level], Channel);
		return Result;
	}

	inline image copy
	(
		image const & SrcImage,
		image::level_type const & SrcLevel,
		glm::uvec2 const & SrcPosition,
		glm::uvec2 const & SrcDimensions,
		image & DstMipmap,
		image::level_type const & DstLevel,
		glm::uvec2 const & DstDimensions
	)
	{
		detail::copy(
			SrcImage[SrcLevel],
			SrcPosition,
			SrcDimensions,
			DstMipmap[DstLevel],
			DstDimensions);
		return DstMipmap;
	}

	//inline mipmap operator+(mipmap const & MipmapA, mipmap const & MipmapB)
	//{
	//
	//}

	//inline mipmap operator-(mipmap const & MipmapA, mipmap const & MipmapB)
	//{
	//
	//}

	//inline mipmap operator*(mipmap const & MipmapA, mipmap const & MipmapB)
	//{
	//
	//}

	//inline mipmap operator/(mipmap const & MipmapA, mipmap const & MipmapB)
	//{
	//
	//}

}//namespace gli
<commit_msg>Fixed GCC build<commit_after>#include <cstring>

namespace gli
{
	namespace detail
	{
		inline image::mipmap duplicate(image::mipmap const & Mipmap2D)
		{
			image::mipmap Result(Mipmap2D.dimensions(), Mipmap2D.format());
			memcpy(Result.data(), Mipmap2D.data(), Mipmap2D.capacity());
			return Result;
		}

		inline image::mipmap flip(image::mipmap const & Mipmap2D)
		{
			image::mipmap Result(Mipmap2D.dimensions(), Mipmap2D.format());

			std::size_t ValueSize = Result.value_size();
			glm::byte * DstPtr = Result.data();
			glm::byte const * const SrcPtr = Mipmap2D.data();

			for(std::size_t j = 0; j < Result.dimensions().y; ++j)
			for(std::size_t i = 0; i < Result.dimensions().x; ++i)
			{
				std::size_t DstIndex = (i + j * Result.dimensions().y) * ValueSize;
				std::size_t SrcIndex = (i + (Result.dimensions().y - j) * Result.dimensions().x) * ValueSize;
				memcpy(DstPtr + DstIndex, SrcPtr + SrcIndex, ValueSize);
			}

			return Result;
		}

		inline image::mipmap mirror(image::mipmap const & Mipmap2D)
		{
			image::mipmap Result(Mipmap2D.dimensions(), Mipmap2D.format());

			std::size_t ValueSize = Mipmap2D.value_size();
			glm::byte * DstPtr = Result.data();
			glm::byte const * const SrcPtr = Mipmap2D.data();

			for(std::size_t j = 0; j < Result.dimensions().y; ++j)
			for(std::size_t i = 0; i < Result.dimensions().x; ++i)
			{
				std::size_t DstIndex = (i + j * Result.dimensions().x) * ValueSize;
				std::size_t SrcIndex = ((Result.dimensions().x - i) + j * Result.dimensions().x) * ValueSize;
				memcpy(DstPtr + DstIndex, SrcPtr + SrcIndex, ValueSize);
			}

			return Result;
		}

		inline image::mipmap swizzle
		(
			image::mipmap const & Mipmap,
			glm::uvec4 const & Channel
		)
		{
			image::mipmap Result = detail::duplicate(Mipmap);

			glm::byte * DataDst = Result.data();
			glm::byte const * const DataSrc = Mipmap.data();

			gli::image::size_type CompSize = Mipmap.value_size() / Mipmap.components();
			gli::image::size_type TexelCount = Mipmap.capacity() / Mipmap.value_size();

			for(gli::image::size_type t = 0; t < TexelCount; ++t)
			for(gli::image::size_type c = 0; c < Mipmap.components(); ++c)
			{
				gli::image::size_type IndexSrc = t * Mipmap.components() + Channel[glm::uvec4::size_type(c)];
				gli::image::size_type IndexDst = t * Mipmap.components() + c;

				memcpy(DataDst + IndexDst, DataSrc + IndexSrc, CompSize);
			}

			return Result;
		}

		inline image::mipmap crop
		(
			image::mipmap const & Mipmap,
			glm::uvec2 const & Position,
			glm::uvec2 const & Size
		)
		{
			assert((Position.x + Size.x) <= Mipmap.dimensions().x && (Position.y + Size.y) <= Mipmap.dimensions().y);

			image::mipmap Result(glm::uvec3(Size, glm::uint(1)), Mipmap.format());

			glm::byte* DstData = Result.data();
			glm::byte const * const SrcData = Mipmap.data();

			for(std::size_t j = 0; j < Size.y; ++j)
			{
				std::size_t DstIndex = 0                                + (0          + j) * Size.x         * Mipmap.value_size();
				std::size_t SrcIndex = Position.x * Mipmap.value_size() + (Position.y + j) * Mipmap.dimensions().x * Mipmap.value_size();
				memcpy(DstData + DstIndex, SrcData + SrcIndex, Mipmap.value_size() * Size.x);
			}

			return Result;
		}

		inline image::mipmap copy
		(
			image::mipmap const & SrcMipmap,
			glm::uvec2 const & SrcPosition,
			glm::uvec2 const & SrcSize,
			image::mipmap & DstMipmap,
			glm::uvec2 const & DstPosition
		)
		{
			assert((SrcPosition.x + SrcSize.x) <= SrcMipmap.dimensions().x && (SrcPosition.y + SrcSize.y) <= SrcMipmap.dimensions().y);
			assert(SrcMipmap.format() == DstMipmap.format());

			glm::byte * DstData = DstMipmap.data();
			glm::byte const * const SrcData = SrcMipmap.data();

			std::size_t SizeX = std::min(std::size_t(SrcSize.x + SrcPosition.x), std::size_t(DstMipmap.dimensions().x  + DstPosition.x));
			std::size_t SizeY = std::min(std::size_t(SrcSize.y + SrcPosition.y), std::size_t(DstMipmap.dimensions().y + DstPosition.y));

			for(std::size_t j = 0; j < SizeY; ++j)
			{
				std::size_t DstIndex = DstPosition.x * DstMipmap.value_size() + (DstPosition.y + j) * DstMipmap.dimensions().x * DstMipmap.value_size();
				std::size_t SrcIndex = SrcPosition.x * SrcMipmap.value_size() + (SrcPosition.y + j) * SrcMipmap.dimensions().x * SrcMipmap.value_size();
				memcpy(DstData + DstIndex, SrcData + SrcIndex, SrcMipmap.value_size() * SizeX);
			}

			return DstMipmap;
		}

	}//namespace detail

	inline image duplicate(image const & Image2D)
	{
		image Result(Image2D.levels());
		for(std::size_t Level = 0; Level < Image2D.levels(); ++Level)
			Result[Level] = detail::duplicate(Image2D[Level]);
		return Result;
	}

	inline image flip(image const & Image2D)
	{
		image Result(Image2D.levels());
		for(std::size_t Level = 0; Level < Image2D.levels(); ++Level)
			Result[Level] = detail::flip(Image2D[Level]);
		return Result;
	}

	inline image mirror(image const & Image2D)
	{
		image Result(Image2D.levels());
		for(std::size_t Level = 0; Level < Image2D.levels(); ++Level)
			Result[Level] = detail::mirror(Image2D[Level]);
		return Result;
	}

	inline image crop
	(
		image const & Image,
		glm::uvec2 const & Position,
		glm::uvec2 const & Size
	)
	{
		image Result(Image.levels());
		for(glm::uint Level = 0; Level < Image.levels(); ++Level)
			Result[Level] = detail::crop(Image[Level], Position >> Level, Size >> Level);
		return Result;
	}

	inline image swizzle
	(
		image const & Image,
		glm::uvec4 const & Channel
	)
	{
		image Result(Image.levels());
		for(std::size_t Level = 0; Level < Image.levels(); ++Level)
			Result[Level] = detail::swizzle(Image[Level], Channel);
		return Result;
	}

	inline image copy
	(
		image const & SrcImage,
		image::level_type const & SrcLevel,
		glm::uvec2 const & SrcPosition,
		glm::uvec2 const & SrcDimensions,
		image & DstMipmap,
		image::level_type const & DstLevel,
		glm::uvec2 const & DstDimensions
	)
	{
		detail::copy(
			SrcImage[SrcLevel],
			SrcPosition,
			SrcDimensions,
			DstMipmap[DstLevel],
			DstDimensions);
		return DstMipmap;
	}

	//inline mipmap operator+(mipmap const & MipmapA, mipmap const & MipmapB)
	//{
	//
	//}

	//inline mipmap operator-(mipmap const & MipmapA, mipmap const & MipmapB)
	//{
	//
	//}

	//inline mipmap operator*(mipmap const & MipmapA, mipmap const & MipmapB)
	//{
	//
	//}

	//inline mipmap operator/(mipmap const & MipmapA, mipmap const & MipmapB)
	//{
	//
	//}

}//namespace gli
<|endoftext|>


<commit_before>/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2020-2020. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#include "beam_asm.hpp"

extern "C"
{
#include "bif.h"
#include "beam_common.h"
#include "code_ix.h"
#include "export.h"
}

const uint8_t *BeamAssembler::nops[3] = {nop1, nop2, nop3};
const uint8_t BeamAssembler::nop1[1] = {0x90};
const uint8_t BeamAssembler::nop2[2] = {0x66, 0x90};
const uint8_t BeamAssembler::nop3[3] = {0x0F, 0x1F, 0x00};

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
static Process *erts_debug_schedule(ErtsSchedulerData *esdp,
                                    Process *c_p,
                                    int calls) {
    PROCESS_MAIN_CHK_LOCKS(c_p);
    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
    c_p = erts_schedule(esdp, c_p, calls);
    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
    ERTS_REQ_PROC_MAIN_LOCK(c_p);
    PROCESS_MAIN_CHK_LOCKS(c_p);
    return c_p;
}
#endif

/* void process_main(ErtsSchedulerData *esdp); */
void BeamGlobalAssembler::emit_process_main() {
    Label context_switch_local = a.newLabel(),
          context_switch_simplified_local = a.newLabel(),
          do_schedule_local = a.newLabel(), schedule_next = a.newLabel();

    const x86::Mem start_time_i =
            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time_i));
    const x86::Mem start_time =
            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time));

    /* Allocate the register structure on the stack to allow computing the
     * runtime stack address from it, greatly reducing the cost of stack
     * swapping. */
    a.sub(x86::rsp, imm(sizeof(ErtsSchedulerRegisters) + ERTS_CACHE_LINE_SIZE));
    a.and_(x86::rsp, imm(~ERTS_CACHE_LINE_MASK));

    a.mov(x86::qword_ptr(ARG1, offsetof(ErtsSchedulerData, registers)),
          x86::rsp);

    /* Center `registers` at the base of x_reg_array so we can use negative
     * 8-bit displacement to address the commonly used aux_regs, located at the
     * start of the ErtsSchedulerRegisters struct. */
    a.lea(registers,
          x86::qword_ptr(x86::rsp,
                         offsetof(ErtsSchedulerRegisters, x_reg_array.d)));

    load_erl_bits_state(ARG1);
    runtime_call<1>(erts_bits_init_state);

#if defined(DEBUG) && defined(NATIVE_ERLANG_STACK)
    /* Save stack bounds so they can be tested without clobbering anything. */
    runtime_call<0>(erts_get_stacklimit);

    a.mov(getSchedulerRegRef(
                  offsetof(ErtsSchedulerRegisters, runtime_stack_end)),
          RET);
    a.mov(getSchedulerRegRef(
                  offsetof(ErtsSchedulerRegisters, runtime_stack_start)),
          x86::rsp);
#elif !defined(NATIVE_ERLANG_STACK)
    /* Save the initial SP of the thread so that we can verify that it
     * doesn't grow. */
#    ifdef JIT_HARD_DEBUG
    a.mov(getInitialSPRef(), x86::rsp);
#    endif

    /* Manually do an `emit_enter_runtime` to match the `emit_leave_runtime`
     * below. We avoid `emit_enter_runtime` because it may do additional
     * assertions that may currently fail.
     *
     * IMPORTANT: We must ensure that this sequence leaves the stack
     * aligned on a 16-byte boundary. */
    a.mov(getRuntimeStackRef(), x86::rsp);
    a.sub(x86::rsp, imm(15));
    a.and_(x86::rsp, imm(-16));
#endif

    a.mov(start_time_i, imm(0));
    a.mov(start_time, imm(0));

    mov_imm(c_p, 0);
    mov_imm(FCALLS, 0);
    mov_imm(ARG3, 0); /* Set reds_used for erts_schedule call */

    a.jmp(schedule_next);

    a.bind(do_schedule_local);
    {
        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])));
        a.sub(ARG3, FCALLS);

        a.jmp(schedule_next);
    }

    a.bind(context_switch_local);
    comment("Context switch, unknown arity/MFA");
    {
        Sint arity_offset = offsetof(ErtsCodeMFA, arity) - sizeof(ErtsCodeMFA);

        a.mov(ARG1, x86::qword_ptr(ARG3, arity_offset));
        a.mov(x86::qword_ptr(c_p, offsetof(Process, arity)), ARG1);

        a.lea(ARG1, x86::qword_ptr(ARG3, -(Sint)sizeof(ErtsCodeMFA)));
        a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), ARG1);

        /* !! Fall through !! */
    }

    a.bind(context_switch_simplified_local);
    comment("Context switch, known arity and MFA");
    {
        Label not_exiting = a.newLabel();

#ifdef ERLANG_FRAME_POINTERS
        /* Kill the current frame pointer to avoid confusing `perf` and similar
         * tools. */
        a.sub(frame_pointer, frame_pointer);
#endif

#ifdef DEBUG
        Label check_i = a.newLabel();
        /* Check that ARG3 is set to a valid CP. */
        a.test(ARG3, imm(_CPMASK));
        a.je(check_i);
        a.ud2();
        a.bind(check_i);
#endif

        a.mov(x86::qword_ptr(c_p, offsetof(Process, i)), ARG3);

#if defined(JIT_HARD_DEBUG) && defined(ERLANG_FRAME_POINTERS)
        a.mov(ARG1, c_p);
        a.mov(ARG2, x86::qword_ptr(c_p, offsetof(Process, frame_pointer)));
        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, stop)));

        runtime_call<3>(erts_validate_stack);
#endif

#ifdef WIN32
        a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.value)));
#else
        a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.counter)));
#endif

        a.test(ARG1d, imm(ERTS_PSFLG_EXITING));
        a.short_().je(not_exiting);
        {
            comment("Process exiting");

            /* We load the beam_exit from memory because it has not yet been set
             * when the global assembler is created. */
            a.mov(ARG1, imm(&beam_exit));
            a.mov(ARG1, x86::qword_ptr(ARG1));
            a.mov(x86::qword_ptr(c_p, offsetof(Process, i)), ARG1);
            a.mov(x86::qword_ptr(c_p, offsetof(Process, arity)), imm(0));
            a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), imm(0));
            a.jmp(do_schedule_local);
        }
        a.bind(not_exiting);

        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])));
        a.sub(ARG3, FCALLS);

        /* Spill reds_used to FCALLS as we no longer need that value */
        a.mov(FCALLS, ARG3);

        a.mov(ARG1, c_p);
        load_x_reg_array(ARG2);
        runtime_call<2>(copy_out_registers);

        /* Restore reds_used from FCALLS */
        a.mov(ARG3, FCALLS);

        /* !! Fall through !! */
    }

    a.bind(schedule_next);
    comment("schedule_next");
    {
        Label schedule = a.newLabel(), skip_long_schedule = a.newLabel();

        /* ARG3 contains reds_used at this point */

        a.cmp(start_time, imm(0));
        a.short_().je(schedule);
        {
            a.mov(ARG1, c_p);
            a.mov(ARG2, start_time);

            /* Spill reds_used in start_time slot */
            a.mov(start_time, ARG3);

            a.mov(ARG3, start_time_i);
            runtime_call<3>(check_monitor_long_schedule);

            /* Restore reds_used */
            a.mov(ARG3, start_time);
        }
        a.bind(schedule);

#ifdef ERLANG_FRAME_POINTERS
        if (erts_frame_layout == ERTS_FRAME_LAYOUT_FP_RA) {
            /* Kill the current frame pointer so that misc jobs that execute
             * during `erts_schedule` aren't attributed to the function we
             * were scheduled out of. */
            a.sub(frame_pointer, frame_pointer);
        }
#endif

        mov_imm(ARG1, 0);
        a.mov(ARG2, c_p);
#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
        runtime_call<3>(erts_debug_schedule);
#else
        runtime_call<3>(erts_schedule);
#endif
        a.mov(c_p, RET);

#ifdef ERTS_MSACC_EXTENDED_STATES
        a.lea(ARG1,
              x86::qword_ptr(registers,
                             offsetof(ErtsSchedulerRegisters,
                                      aux_regs.d.erts_msacc_cache)));
        runtime_call<1>(erts_msacc_update_cache);
#endif

        a.mov(ARG1, imm((UWord)&erts_system_monitor_long_schedule));
        a.cmp(x86::qword_ptr(ARG1), imm(0));
        a.mov(start_time, imm(0));
        a.short_().je(skip_long_schedule);
        {
            /* Enable long schedule test */
            runtime_call<0>(erts_timestamp_millis);
            a.mov(start_time, RET);
            a.mov(RET, x86::qword_ptr(c_p, offsetof(Process, i)));
            a.mov(start_time_i, RET);
        }
        a.bind(skip_long_schedule);

        /* Copy arguments */
        a.mov(ARG1, c_p);
        load_x_reg_array(ARG2);
        runtime_call<2>(copy_in_registers);

        /* Setup reduction counting */
        a.mov(FCALLS, x86::qword_ptr(c_p, offsetof(Process, fcalls)));
        a.mov(x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])), FCALLS);

#ifdef DEBUG
        a.mov(x86::qword_ptr(c_p, offsetof(Process, debug_reds_in)), FCALLS);
#endif

        /* Check whether save calls is on */
        a.mov(ARG1, c_p);
        a.mov(ARG2, imm(ERTS_PSD_SAVED_CALLS_BUF));
        runtime_call<2>(erts_psd_get);

        /* Read the active code index, overriding it with
         * ERTS_SAVE_CALLS_CODE_IX when save_calls is enabled (RET != 0). */
        a.test(RET, RET);
        a.mov(ARG1, imm(&the_active_code_index));
        a.mov(ARG2, imm(ERTS_SAVE_CALLS_CODE_IX));
        a.mov(active_code_ix.r32(), x86::dword_ptr(ARG1));
        a.cmovnz(active_code_ix, ARG2);

        /* Start executing the Erlang process. Note that reductions have
         * already been set up above. */
        emit_leave_runtime<Update::eStack | Update::eHeap>();

        /* Check if we are just returning from a dirty nif/bif call and if so we
         * need to do a bit of cleaning up before continuing. */
        a.mov(RET, x86::qword_ptr(c_p, offsetof(Process, i)));
        a.cmp(x86::qword_ptr(RET), imm(op_call_nif_WWW));
        a.je(labels[dispatch_nif]);
        a.cmp(x86::qword_ptr(RET), imm(op_call_bif_W));
        a.je(labels[dispatch_bif]);
        a.jmp(RET);
    }

    /* Processes may jump to the exported entry points below, executing on the
     * Erlang stack when entering. These are separate from the `_local` labels
     * above as we don't want to worry about which stack we're on when the
     * cases overlap. */

    /* `ga->get_context_switch()`
     *
     * The *next* instruction pointer is provided in ARG3, and must be preceded
     * by an ErtsCodeMFA. */
    a.bind(labels[context_switch]);
    {
        emit_enter_runtime<Update::eStack | Update::eHeap>();

        a.jmp(context_switch_local);
    }

    /* `ga->get_context_switch_simplified()`
     *
     * The next instruction pointer is provided in ARG3, which does not need to
     * point past an ErtsCodeMFA as the process structure has already been
     * updated. */
    a.bind(labels[context_switch_simplified]);
    {
        emit_enter_runtime<Update::eStack | Update::eHeap>();

        a.jmp(context_switch_simplified_local);
    }

    /* `ga->get_do_schedule()`
     *
     * `c_p->i` must be set prior to jumping here. */
    a.bind(labels[do_schedule]);
    {
        emit_enter_runtime<Update::eStack | Update::eHeap>();

        a.jmp(do_schedule_local);
    }
}
<commit_msg>jit: Compute the address of beam_exit instead of loading it (x86)<commit_after>/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2020-2020. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#include "beam_asm.hpp"

extern "C"
{
#include "bif.h"
#include "beam_common.h"
#include "code_ix.h"
#include "export.h"
}

const uint8_t *BeamAssembler::nops[3] = {nop1, nop2, nop3};
const uint8_t BeamAssembler::nop1[1] = {0x90};
const uint8_t BeamAssembler::nop2[2] = {0x66, 0x90};
const uint8_t BeamAssembler::nop3[3] = {0x0F, 0x1F, 0x00};

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
static Process *erts_debug_schedule(ErtsSchedulerData *esdp,
                                    Process *c_p,
                                    int calls) {
    PROCESS_MAIN_CHK_LOCKS(c_p);
    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
    c_p = erts_schedule(esdp, c_p, calls);
    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
    ERTS_REQ_PROC_MAIN_LOCK(c_p);
    PROCESS_MAIN_CHK_LOCKS(c_p);
    return c_p;
}
#endif

/* void process_main(ErtsSchedulerData *esdp); */
void BeamGlobalAssembler::emit_process_main() {
    Label context_switch_local = a.newLabel(),
          context_switch_simplified_local = a.newLabel(),
          do_schedule_local = a.newLabel(), schedule_next = a.newLabel();

    const x86::Mem start_time_i =
            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time_i));
    const x86::Mem start_time =
            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time));

    /* Allocate the register structure on the stack to allow computing the
     * runtime stack address from it, greatly reducing the cost of stack
     * swapping. */
    a.sub(x86::rsp, imm(sizeof(ErtsSchedulerRegisters) + ERTS_CACHE_LINE_SIZE));
    a.and_(x86::rsp, imm(~ERTS_CACHE_LINE_MASK));

    a.mov(x86::qword_ptr(ARG1, offsetof(ErtsSchedulerData, registers)),
          x86::rsp);

    /* Center `registers` at the base of x_reg_array so we can use negative
     * 8-bit displacement to address the commonly used aux_regs, located at the
     * start of the ErtsSchedulerRegisters struct. */
    a.lea(registers,
          x86::qword_ptr(x86::rsp,
                         offsetof(ErtsSchedulerRegisters, x_reg_array.d)));

    load_erl_bits_state(ARG1);
    runtime_call<1>(erts_bits_init_state);

#if defined(DEBUG) && defined(NATIVE_ERLANG_STACK)
    /* Save stack bounds so they can be tested without clobbering anything. */
    runtime_call<0>(erts_get_stacklimit);

    a.mov(getSchedulerRegRef(
                  offsetof(ErtsSchedulerRegisters, runtime_stack_end)),
          RET);
    a.mov(getSchedulerRegRef(
                  offsetof(ErtsSchedulerRegisters, runtime_stack_start)),
          x86::rsp);
#elif !defined(NATIVE_ERLANG_STACK)
    /* Save the initial SP of the thread so that we can verify that it
     * doesn't grow. */
#    ifdef JIT_HARD_DEBUG
    a.mov(getInitialSPRef(), x86::rsp);
#    endif

    /* Manually do an `emit_enter_runtime` to match the `emit_leave_runtime`
     * below. We avoid `emit_enter_runtime` because it may do additional
     * assertions that may currently fail.
     *
     * IMPORTANT: We must ensure that this sequence leaves the stack
     * aligned on a 16-byte boundary. */
    a.mov(getRuntimeStackRef(), x86::rsp);
    a.sub(x86::rsp, imm(15));
    a.and_(x86::rsp, imm(-16));
#endif

    a.mov(start_time_i, imm(0));
    a.mov(start_time, imm(0));

    mov_imm(c_p, 0);
    mov_imm(FCALLS, 0);
    mov_imm(ARG3, 0); /* Set reds_used for erts_schedule call */

    a.jmp(schedule_next);

    a.bind(do_schedule_local);
    {
        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])));
        a.sub(ARG3, FCALLS);

        a.jmp(schedule_next);
    }

    a.bind(context_switch_local);
    comment("Context switch, unknown arity/MFA");
    {
        Sint arity_offset = offsetof(ErtsCodeMFA, arity) - sizeof(ErtsCodeMFA);

        a.mov(ARG1, x86::qword_ptr(ARG3, arity_offset));
        a.mov(x86::qword_ptr(c_p, offsetof(Process, arity)), ARG1);

        a.lea(ARG1, x86::qword_ptr(ARG3, -(Sint)sizeof(ErtsCodeMFA)));
        a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), ARG1);

        /* !! Fall through !! */
    }

    a.bind(context_switch_simplified_local);
    comment("Context switch, known arity and MFA");
    {
        Label not_exiting = a.newLabel();

#ifdef ERLANG_FRAME_POINTERS
        /* Kill the current frame pointer to avoid confusing `perf` and similar
         * tools. */
        a.sub(frame_pointer, frame_pointer);
#endif

#ifdef DEBUG
        Label check_i = a.newLabel();
        /* Check that ARG3 is set to a valid CP. */
        a.test(ARG3, imm(_CPMASK));
        a.je(check_i);
        a.ud2();
        a.bind(check_i);
#endif

        a.mov(x86::qword_ptr(c_p, offsetof(Process, i)), ARG3);

#if defined(JIT_HARD_DEBUG) && defined(ERLANG_FRAME_POINTERS)
        a.mov(ARG1, c_p);
        a.mov(ARG2, x86::qword_ptr(c_p, offsetof(Process, frame_pointer)));
        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, stop)));

        runtime_call<3>(erts_validate_stack);
#endif

#ifdef WIN32
        a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.value)));
#else
        a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.counter)));
#endif

        a.test(ARG1d, imm(ERTS_PSFLG_EXITING));
        a.short_().je(not_exiting);
        {
            comment("Process exiting");

            a.lea(ARG1, x86::qword_ptr(labels[process_exit]));
            a.mov(x86::qword_ptr(c_p, offsetof(Process, i)), ARG1);
            a.mov(x86::qword_ptr(c_p, offsetof(Process, arity)), imm(0));
            a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), imm(0));
            a.jmp(do_schedule_local);
        }
        a.bind(not_exiting);

        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])));
        a.sub(ARG3, FCALLS);

        /* Spill reds_used to FCALLS as we no longer need that value */
        a.mov(FCALLS, ARG3);

        a.mov(ARG1, c_p);
        load_x_reg_array(ARG2);
        runtime_call<2>(copy_out_registers);

        /* Restore reds_used from FCALLS */
        a.mov(ARG3, FCALLS);

        /* !! Fall through !! */
    }

    a.bind(schedule_next);
    comment("schedule_next");
    {
        Label schedule = a.newLabel(), skip_long_schedule = a.newLabel();

        /* ARG3 contains reds_used at this point */

        a.cmp(start_time, imm(0));
        a.short_().je(schedule);
        {
            a.mov(ARG1, c_p);
            a.mov(ARG2, start_time);

            /* Spill reds_used in start_time slot */
            a.mov(start_time, ARG3);

            a.mov(ARG3, start_time_i);
            runtime_call<3>(check_monitor_long_schedule);

            /* Restore reds_used */
            a.mov(ARG3, start_time);
        }
        a.bind(schedule);

#ifdef ERLANG_FRAME_POINTERS
        if (erts_frame_layout == ERTS_FRAME_LAYOUT_FP_RA) {
            /* Kill the current frame pointer so that misc jobs that execute
             * during `erts_schedule` aren't attributed to the function we
             * were scheduled out of. */
            a.sub(frame_pointer, frame_pointer);
        }
#endif

        mov_imm(ARG1, 0);
        a.mov(ARG2, c_p);
#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
        runtime_call<3>(erts_debug_schedule);
#else
        runtime_call<3>(erts_schedule);
#endif
        a.mov(c_p, RET);

#ifdef ERTS_MSACC_EXTENDED_STATES
        a.lea(ARG1,
              x86::qword_ptr(registers,
                             offsetof(ErtsSchedulerRegisters,
                                      aux_regs.d.erts_msacc_cache)));
        runtime_call<1>(erts_msacc_update_cache);
#endif

        a.mov(ARG1, imm((UWord)&erts_system_monitor_long_schedule));
        a.cmp(x86::qword_ptr(ARG1), imm(0));
        a.mov(start_time, imm(0));
        a.short_().je(skip_long_schedule);
        {
            /* Enable long schedule test */
            runtime_call<0>(erts_timestamp_millis);
            a.mov(start_time, RET);
            a.mov(RET, x86::qword_ptr(c_p, offsetof(Process, i)));
            a.mov(start_time_i, RET);
        }
        a.bind(skip_long_schedule);

        /* Copy arguments */
        a.mov(ARG1, c_p);
        load_x_reg_array(ARG2);
        runtime_call<2>(copy_in_registers);

        /* Setup reduction counting */
        a.mov(FCALLS, x86::qword_ptr(c_p, offsetof(Process, fcalls)));
        a.mov(x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])), FCALLS);

#ifdef DEBUG
        a.mov(x86::qword_ptr(c_p, offsetof(Process, debug_reds_in)), FCALLS);
#endif

        /* Check whether save calls is on */
        a.mov(ARG1, c_p);
        a.mov(ARG2, imm(ERTS_PSD_SAVED_CALLS_BUF));
        runtime_call<2>(erts_psd_get);

        /* Read the active code index, overriding it with
         * ERTS_SAVE_CALLS_CODE_IX when save_calls is enabled (RET != 0). */
        a.test(RET, RET);
        a.mov(ARG1, imm(&the_active_code_index));
        a.mov(ARG2, imm(ERTS_SAVE_CALLS_CODE_IX));
        a.mov(active_code_ix.r32(), x86::dword_ptr(ARG1));
        a.cmovnz(active_code_ix, ARG2);

        /* Start executing the Erlang process. Note that reductions have
         * already been set up above. */
        emit_leave_runtime<Update::eStack | Update::eHeap>();

        /* Check if we are just returning from a dirty nif/bif call and if so we
         * need to do a bit of cleaning up before continuing. */
        a.mov(RET, x86::qword_ptr(c_p, offsetof(Process, i)));
        a.cmp(x86::qword_ptr(RET), imm(op_call_nif_WWW));
        a.je(labels[dispatch_nif]);
        a.cmp(x86::qword_ptr(RET), imm(op_call_bif_W));
        a.je(labels[dispatch_bif]);
        a.jmp(RET);
    }

    /* Processes may jump to the exported entry points below, executing on the
     * Erlang stack when entering. These are separate from the `_local` labels
     * above as we don't want to worry about which stack we're on when the
     * cases overlap. */

    /* `ga->get_context_switch()`
     *
     * The *next* instruction pointer is provided in ARG3, and must be preceded
     * by an ErtsCodeMFA. */
    a.bind(labels[context_switch]);
    {
        emit_enter_runtime<Update::eStack | Update::eHeap>();

        a.jmp(context_switch_local);
    }

    /* `ga->get_context_switch_simplified()`
     *
     * The next instruction pointer is provided in ARG3, which does not need to
     * point past an ErtsCodeMFA as the process structure has already been
     * updated. */
    a.bind(labels[context_switch_simplified]);
    {
        emit_enter_runtime<Update::eStack | Update::eHeap>();

        a.jmp(context_switch_simplified_local);
    }

    /* `ga->get_do_schedule()`
     *
     * `c_p->i` must be set prior to jumping here. */
    a.bind(labels[do_schedule]);
    {
        emit_enter_runtime<Update::eStack | Update::eHeap>();

        a.jmp(do_schedule_local);
    }
}
<|endoftext|>


<commit_before>/* Copyright 2019 Benjamin Worpitz
 *
 * This file is part of Alpaka.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#pragma once

#ifdef ALPAKA_ACC_CPU_BT_OMP4_ENABLED

#if _OPENMP < 201307
    #error If ALPAKA_ACC_CPU_BT_OMP4_ENABLED is set, the compiler has to support OpenMP 4.0 or higher!
#endif

#include <alpaka/pltf/Traits.hpp>
#include <alpaka/dev/DevOmp4.hpp>
#include <alpaka/core/Concepts.hpp>

#include <sstream>
#include <vector>

namespace alpaka
{
    namespace pltf
    {
        //#############################################################################
        //! The CPU device platform.
        class PltfOmp4 :
            public concepts::Implements<ConceptPltf, PltfOmp4>
        {
        public:
            //-----------------------------------------------------------------------------
            ALPAKA_FN_HOST PltfOmp4() = delete;
        };
    }

    namespace dev
    {
        namespace traits
        {
            //#############################################################################
            //! The CPU device device type trait specialization.
            template<>
            struct DevType<
                pltf::PltfOmp4>
            {
                using type = dev::DevOmp4;
            };
        }
    }
    namespace pltf
    {
        namespace traits
        {
            //#############################################################################
            //! The CPU platform device count get trait specialization.
            template<>
            struct GetDevCount<
                pltf::PltfOmp4>
            {
                //-----------------------------------------------------------------------------
                ALPAKA_FN_HOST static auto getDevCount()
                -> std::size_t
                {
                    ALPAKA_DEBUG_FULL_LOG_SCOPE;

                    return static_cast<std::size_t>(::omp_get_num_devices());
                }
            };

            //#############################################################################
            //! The CPU platform device get trait specialization.
            template<>
            struct GetDevByIdx<
                pltf::PltfOmp4>
            {
                //-----------------------------------------------------------------------------
                //! \param devIdx device id, less than GetDevCount, will be set to omp_get_initial_device() if < 0
                ALPAKA_FN_HOST static auto getDevByIdx(
                    int devIdx)
                -> dev::DevOmp4
                {
                    ALPAKA_DEBUG_FULL_LOG_SCOPE;

                    int const devCount(pltf::getDevCount<pltf::PltfOmp4>());
                    if(devIdx >= devCount)
                    {
                        std::stringstream ssErr;
                        ssErr << "Unable to return device handle for CPU device with index " << devIdx << " because there are only " << devCount << " devices!";
                        throw std::runtime_error(ssErr.str());
                    }
                    else if(devIdx < 0)
                    {
                        devIdx = ::omp_get_initial_device();
                    }

                    return {devIdx};
                }
            };
        }
    }
}

#endif
<commit_msg>PltfOmp4: Fix GetDevByIdx(devIdx) cast warning<commit_after>/* Copyright 2019 Benjamin Worpitz
 *
 * This file is part of Alpaka.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#pragma once

#ifdef ALPAKA_ACC_CPU_BT_OMP4_ENABLED

#if _OPENMP < 201307
    #error If ALPAKA_ACC_CPU_BT_OMP4_ENABLED is set, the compiler has to support OpenMP 4.0 or higher!
#endif

#include <alpaka/pltf/Traits.hpp>
#include <alpaka/dev/DevOmp4.hpp>
#include <alpaka/core/Concepts.hpp>

#include <sstream>
#include <vector>
#include <limits>

namespace alpaka
{
    namespace pltf
    {
        //#############################################################################
        //! The CPU device platform.
        class PltfOmp4 :
            public concepts::Implements<ConceptPltf, PltfOmp4>
        {
        public:
            //-----------------------------------------------------------------------------
            ALPAKA_FN_HOST PltfOmp4() = delete;
        };
    }

    namespace dev
    {
        namespace traits
        {
            //#############################################################################
            //! The CPU device device type trait specialization.
            template<>
            struct DevType<
                pltf::PltfOmp4>
            {
                using type = dev::DevOmp4;
            };
        }
    }
    namespace pltf
    {
        namespace traits
        {
            //#############################################################################
            //! The CPU platform device count get trait specialization.
            template<>
            struct GetDevCount<
                pltf::PltfOmp4>
            {
                //-----------------------------------------------------------------------------
                ALPAKA_FN_HOST static auto getDevCount()
                -> std::size_t
                {
                    ALPAKA_DEBUG_FULL_LOG_SCOPE;

                    return static_cast<std::size_t>(::omp_get_num_devices());
                }
            };

            //#############################################################################
            //! The CPU platform device get trait specialization.
            template<>
            struct GetDevByIdx<
                pltf::PltfOmp4>
            {
                //-----------------------------------------------------------------------------
                //! \param devIdx device id, less than GetDevCount, will be set to omp_get_initial_device() if < 0
                ALPAKA_FN_HOST static auto getDevByIdx(
                    std::size_t devIdx)
                -> dev::DevOmp4
                {
                    ALPAKA_DEBUG_FULL_LOG_SCOPE;

                    std::size_t const devCount(static_cast<std::size_t>(pltf::getDevCount<pltf::PltfOmp4>()));
                    int devIdxOmp4 = static_cast<int>(devIdx);
                    if(devIdx >= devCount)
                    { // devIdx param must be unsigned, take take this case to use the initial device
                        devIdxOmp4 = ::omp_get_initial_device();
                    }

                    return {devIdxOmp4};
                }
            };
        }
    }
}

#endif
<|endoftext|>