Establish JVM opts, adjusting memory for the context if needed.

    This allows using less or more memory for highly parallel or multicore
    supporting processes, respectively.
179117
def adjust_opts(in_opts, config):
    
    memory_adjust = config["algorithm"].get("memory_adjust", {})
    out_opts = []
    for opt in in_opts:
        if opt.startswith("-Xmx") or (opt.startswith("-Xms") and memory_adjust.get("direction") == "decrease"):
            arg = opt[:4]
            opt = "{arg}{val}".format(arg=arg,
                                      val=adjust_memory(opt[4:],
                                                        memory_adjust.get("magnitude", 1),
                                                        memory_adjust.get("direction"),
                                                        maximum=memory_adjust.get("maximum")))
        out_opts.append(opt)
    return out_opts
1105531
def allocate_stack(size=DEFAULT_STACK_SIZE):
    
    # Allocate memory with appropriate flags for a stack as in https://blog.fefe.de/?ts=a85c8ba7
    base = libc.mmap(
        None,
        size + GUARD_PAGE_SIZE,
        libc.PROT_READ | libc.PROT_WRITE,
        libc.MAP_PRIVATE | libc.MAP_ANONYMOUS | libc.MAP_GROWSDOWN | libc.MAP_STACK,
        -1, 0)

    try:
        # create a guard page that crashes the application when it is written to (on stack overflow)
        libc.mprotect(base, GUARD_PAGE_SIZE, libc.PROT_NONE)

        yield ctypes.c_void_p(base + size + GUARD_PAGE_SIZE)
    finally:
        libc.munmap(base, size + GUARD_PAGE_SIZE)
990751
private void placeVMs(Parameters ps, List<AbstractStrategy<?>> strategies, List<VMTransition> actions, OnStableNodeFirst schedHeuristic, Map<IntVar, VM> map) {
        IntValueSelector rnd = new WorstFit(map, rp, new BiggestDimension());
        if (!useResources) {
            rnd = new RandomVMPlacement(rp, map, true, ps.getRandomSeed());
        }
        IntVar[] hosts = dSlices(actions).map(Slice::getHoster).filter(v -> !v.isInstantiated()).toArray(IntVar[]::new);
        if (hosts.length > 0) {
            strategies.add(new IntStrategy(hosts, new HostingVariableSelector(rp.getModel(), schedHeuristic), rnd));
        }
    }
1284854
void onNextJVM(final AllocatedEvaluator allocatedEvaluator) {
    try {
      final Configuration contextConfiguration = ContextConfiguration.CONF
          .set(ContextConfiguration.IDENTIFIER, "HelloREEFContext")
          .build();

      final Configuration taskConfiguration = TaskConfiguration.CONF
          .set(TaskConfiguration.IDENTIFIER, "HelloREEFTask")
          .set(TaskConfiguration.TASK, HelloTask.class)
          .build();

      allocatedEvaluator.submitContextAndTask(contextConfiguration, taskConfiguration);
    } catch (final BindException ex) {
      final String message = "Unable to setup Task or Context configuration.";
      LOG.log(Level.SEVERE, message, ex);
      throw new RuntimeException(message, ex);
    }
  }
546383
static void set_goals( String msg, boolean oom , long bytes) {
    // Our best guess of free memory, as of the last GC cycle
    final long heapUsedGC = Cleaner.HEAP_USED_AT_LAST_GC;
    final long timeGC = Cleaner.TIME_AT_LAST_GC;
    final long freeHeap = MEM_MAX - heapUsedGC;
    assert freeHeap >= 0 : "I am really confused about the heap usage; MEM_MAX="+MEM_MAX+" heapUsedGC="+heapUsedGC;
    // Current memory held in the K/V store.
    final long cacheUsageGC = Cleaner.KV_USED_AT_LAST_GC;
    // Our best guess of POJO object usage: Heap_used minus cache used
    final long pojoUsedGC = Math.max(heapUsedGC - cacheUsageGC,0);

    // Block allocations if:
    // the cache is > 7/8 MEM_MAX, OR
    // we cannot allocate an equal amount of POJOs, pojoUsedGC > freeHeap.
    // Decay POJOS_USED by 1/8th every 5 sec: assume we got hit with a single
    // large allocation which is not repeating - so we do not need to have
    // double the POJO amount.
    // Keep at least 1/8th heap for caching.
    // Emergency-clean the cache down to the blocking level.
    long d = MEM_CRITICAL;      // Block-allocation level; cache can grow till this
    // Decay POJO amount
    long p = pojoUsedGC;
    long age = (System.currentTimeMillis() - timeGC); // Age since last FullGC
    age = Math.min(age,10*60*1000 ); // Clip at 10mins
    while( (age-=5000) > 0 ) p = p-(p>>3); // Decay effective POJO by 1/8th every 5sec
    d -= 2*p - bytes; // Allow for the effective POJO, and again to throttle GC rate (and allow for this allocation)
    d = Math.max(d,MEM_MAX>>3); // Keep at least 1/8th heap
    if( Cleaner.DESIRED != -1 ) // Set to -1 only for OOM/Cleaner testing.  Never negative normally
      Cleaner.DESIRED = d;      // Desired caching level
    final long cacheUsageNow = Cleaner.Histo.cached();

    boolean skipThisLogMessageToAvoidSpammingTheLogs = false;
    String m="";
    if( cacheUsageNow > Cleaner.DESIRED ) {
      m = (CAN_ALLOC?"Swapping!  ":"blocked:   ");
      if( oom ) setMemLow(); // Stop allocations; trigger emergency clean
      Cleaner.kick_store_cleaner();
    } else { // Else we are not *emergency* cleaning, but may be lazily cleaning.
      setMemGood();             // Cache is below desired level; unblock allocations
      if( oom ) {               // But still have an OOM?
        m = "Unblock allocations; cache below desired, but also OOM: ";
        // Means the heap is full of uncached POJO's - which cannot be spilled.
        // Here we enter the zone of possibly dieing for OOM.  There's no point
        // in blocking allocations, as no more memory can be freed by more
        // cache-flushing.  Might as well proceed on a "best effort" basis.

        long now = System.currentTimeMillis();
        if ((now - oomLastLogTimestamp) >= SIXTY_SECONDS_IN_MILLIS) {
          oomLastLogTimestamp = now;
        }
        else {
          skipThisLogMessageToAvoidSpammingTheLogs = true;
        }
      } else { 
        m = "MemGood:   "; // Cache is low enough, room for POJO allocation - full steam ahead!
      }
    }

    if (skipThisLogMessageToAvoidSpammingTheLogs) {
      return;
    }

    // No logging if under memory pressure: can deadlock the cleaner thread
    String s = m+msg+", (K/V:"+PrettyPrint.bytes(cacheUsageGC)+" + POJO:"+PrettyPrint.bytes(pojoUsedGC)+" + FREE:"+PrettyPrint.bytes(freeHeap)+" == MEM_MAX:"+PrettyPrint.bytes(MEM_MAX)+"), desiredKV="+PrettyPrint.bytes(Cleaner.DESIRED)+(oom?" OOM!":" NO-OOM");
    if( CAN_ALLOC ) { if( oom ) Log.warn(s); else Log.debug(s); }
    else            System.err.println(s);
  }
63613
private static synchronized void initialize() {
        PlatformProperties pp = PlatformProperties.getPlatformProperties();

        String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();
        String pidString = processName.substring(0, processName.indexOf('@'));
        pid = Integer.valueOf(pidString);
        initialized = true;

        // get the RSS and other stats from scraping "ps" from the command line
        PSScraper.PSData psdata = PSScraper.getPSData(pid);
        assert(psdata.rss > 0);

        // figure out how much memory this thing has
        memorysize = pp.ramInMegabytes;
        assert(memorysize > 0);

        // now try to figure out the best way to get the rss size
        long rss = -1;

        // try the mac method
        try {
            rss = ExecutionEngine.nativeGetRSS();
        }
        // This catch is broad to specifically include the UnsatisfiedLinkError that arises when
        // using the hsqldb backend on linux -- along with any other exceptions that might arise.
        // Otherwise, the hsql backend would get an annoying report to stdout
        // as the useless stats thread got needlessly killed.
        catch (Throwable e) { }
        if (rss > 0) mode = GetRSSMode.MACOSX_NATIVE;

        // try procfs
        rss = getRSSFromProcFS();
        if (rss > 0) mode = GetRSSMode.PROCFS;

        // notify users if stats collection might be slow
        if (mode == GetRSSMode.PS) {
            VoltLogger logger = new VoltLogger("HOST");
            logger.warn("System statistics will be collected in a sub-optimal "
                    + "manner because either procfs couldn't be read from or "
                    + "the native library couldn't be loaded.");
        }
    }
388717
private static void configureCaches(
      ImagePipelineConfig.Builder configBuilder,
      Context context) {
    final MemoryCacheParams bitmapCacheParams = new MemoryCacheParams(
        ConfigConstants.MAX_MEMORY_CACHE_SIZE, // Max total size of elements in the cache
        Integer.MAX_VALUE,                     // Max entries in the cache
        ConfigConstants.MAX_MEMORY_CACHE_SIZE, // Max total size of elements in eviction queue
        Integer.MAX_VALUE,                     // Max length of eviction queue
        Integer.MAX_VALUE,                     // Max cache entry size
        TimeUnit.MINUTES.toMillis(5));         // Interval for checking cache parameters
    configBuilder
        .setBitmapMemoryCacheParamsSupplier(
            new Supplier<MemoryCacheParams>() {
              public MemoryCacheParams get() {
                return bitmapCacheParams;
              }
            })
        .setMainDiskCacheConfig(
            DiskCacheConfig.newBuilder(context)
                .setBaseDirectoryPath(context.getApplicationContext().getCacheDir())
                .setBaseDirectoryName(IMAGE_PIPELINE_CACHE_DIR)
                .setMaxCacheSize(ConfigConstants.MAX_DISK_CACHE_SIZE)
                .build());
  }
757570
public static PipelineOptions tweakOptions(PipelineOptions opts) {
    if (!DataflowRunner.class.isAssignableFrom(opts.getRunner())) {
      return opts;
    }
    DataflowPipelineOptions dataflowOpts = opts.as(DataflowPipelineOptions.class);

    // By default, dataflow allocates 250 GB local disks, thats not necessary. Lower it unless the
    // user requested an explicit size
    if (dataflowOpts.getDiskSizeGb() == 0) {
      dataflowOpts.setDiskSizeGb(25);
    }

    return dataflowOpts;
  }
1230550
def _scale_cores_to_memory(cores, mem_per_core, sysinfo, system_memory):
    
    total_mem = "%.2f" % (cores * mem_per_core + system_memory)
    if "cores" not in sysinfo:
        return cores, total_mem, 1.0

    total_mem = min(float(total_mem), float(sysinfo["memory"]) - system_memory)
    cores = min(cores, int(sysinfo["cores"]))
    mem_cores = int(math.floor(float(total_mem) / mem_per_core))  # cores based on available memory
    if mem_cores < 1:
        out_cores = 1
    elif mem_cores < cores:
        out_cores = mem_cores
    else:
        out_cores = cores
    mem_pct = float(out_cores) / float(cores)
    return out_cores, total_mem, mem_pct
188336
def set_mpi_procs(self, mpi_procs):
        """"""
        QueueAdapter.set_mpi_procs(self, mpi_procs)

        num_nodes, rest_cores = self.hw.divmod_node(mpi_procs, omp_threads=1)
        if num_nodes == 0:
            self.qparams["nodes"] = 1
            self.qparams["ppn"] = mpi_procs
        else:
            if rest_cores != 0:
                # Pack cores as much as possible.
                num_nodes += 1
            self.qparams["nodes"] = num_nodes
            self.qparams["ppn"] = self.hw.cores_per_node
671099
@Setup(Level.Trial)
  public void setup() throws Exception {
    super.setup(clientExecutor,
        ExecutorType.DIRECT,
        MessageSize.SMALL,
        responseSize,
        FlowWindowSize.MEDIUM,
        ChannelType.NIO,
        maxConcurrentStreams,
        channelCount);
    callCounter = new AtomicLong();
    completed = new AtomicBoolean();
    record = new AtomicBoolean();
    latch =
        startFlowControlledStreamingCalls(maxConcurrentStreams, callCounter, record, completed, 1);
  }
106755
protected void addConfiguredProcessorParam(final ProcessorParam param) throws NullPointerException, BuildException {
    if (param == null) {
      throw new NullPointerException("param");
    }
    if (isReference()) {
      throw noChildrenAllowed();
    }
    this.processorParams.addElement(param);
  }
1106899
def optimize(args):
    
    p = OptionParser(optimize.__doc__)
    p.add_option("--skiprecover", default=False, action="store_true",
                 help="Do not import 'recover' contigs")
    p.add_option("--startover", default=False, action="store_true",
                 help="Do not resume from existing tour file")
    p.add_option("--skipGA", default=False, action="store_true",
                 help="Skip GA step")
    p.set_outfile(outfile=None)
    p.set_cpus()
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    clmfile, = args
    startover = opts.startover
    runGA = not opts.skipGA
    cpus = opts.cpus

    # Load contact map
    clm = CLMFile(clmfile, skiprecover=opts.skiprecover)

    tourfile = opts.outfile or clmfile.rsplit(".", 1)[0] + ".tour"
    if startover:
        tourfile = None
    tour = clm.activate(tourfile=tourfile)

    fwtour = open(tourfile, "w")
    # Store INIT tour
    print_tour(fwtour, clm.tour, "INIT",
               clm.active_contigs, clm.oo, signs=clm.signs)

    if runGA:
        for phase in range(1, 3):
            tour = optimize_ordering(fwtour, clm, phase, cpus)
            tour = clm.prune_tour(tour, cpus)

    # Flip orientations
    phase = 1
    while True:
        tag1, tag2 = optimize_orientations(fwtour, clm, phase, cpus)
        if tag1 == REJECT and tag2 == REJECT:
            logging.debug("Terminating ... no more {}".format(ACCEPT))
            break
        phase += 1

    fwtour.close()
148562
public void optimize() throws IOException {

        if ((m_index != null) && LOG.isInfoEnabled()) {
            LOG.info(
                Messages.get().getBundle().key(
                    Messages.LOG_INDEX_WRITER_MSG_OPTIMIZE_2,
                    m_index.getName(),
                    m_index.getPath()));
        }
        int oldPriority = Thread.currentThread().getPriority();
        // we don't want the priority too low as the process should complete as fast as possible
        Thread.currentThread().setPriority(Thread.NORM_PRIORITY / 2);
        m_indexWriter.forceMerge(5);
        Thread.currentThread().setPriority(oldPriority);
    }
191843
def _setup_resources():
    
    target_procs = 10240
    cur_proc, max_proc = resource.getrlimit(resource.RLIMIT_NPROC)
    target_proc = min(max_proc, target_procs) if max_proc > 0 else target_procs
    resource.setrlimit(resource.RLIMIT_NPROC, (max(cur_proc, target_proc), max_proc))
    cur_hdls, max_hdls = resource.getrlimit(resource.RLIMIT_NOFILE)
    target_hdls = min(max_hdls, target_procs) if max_hdls > 0 else target_procs
    resource.setrlimit(resource.RLIMIT_NOFILE, (max(cur_hdls, target_hdls), max_hdls))
569205
def execution_options
      {}.tap do |opts|
        # @todo rework consistency module to be more
        #      abstract implementation for all execution options
        opts[:consistency] = consistency if consistency
        opts[:paging_state] = paging_state if respond_to?(:paging_state) && paging_state
        opts[:page_size] = stateless_page_size if respond_to?(:stateless_page_size) && stateless_page_size
      end
    end
531720
def find_optconf(self, pconfs):
        """"""
        # Save pconfs for future reference.
        self.set_pconfs(pconfs)

        # Select the partition on which we'll be running and set MPI/OMP cores.
        optconf = self.manager.select_qadapter(pconfs)
        return optconf