Get the current GPU usage of available GPUs
357021
def GpuUsage(**kargs):
    
    usage = (False, None)
    gpu_status = {'vent_usage': {'dedicated': [], 'mem_mb': {}}}

    path_dirs = PathDirs(**kargs)
    path_dirs.host_config()
    template = Template(template=path_dirs.cfg_file)

    # get running jobs using gpus
    try:
        d_client = docker.from_env()
        c = d_client.containers.list(all=False,
                                     filters={'label': 'vent-plugin'})
        for container in c:
            if ('vent.gpu' in container.attrs['Config']['Labels'] and
                    container.attrs['Config']['Labels']['vent.gpu'] == 'yes'):
                device = container.attrs['Config']['Labels']['vent.gpu.device']
                if ('vent.gpu.dedicated' in container.attrs['Config']['Labels'] and
                        container.attrs['Config']['Labels']['vent.gpu.dedicated'] == 'yes'):
                    gpu_status['vent_usage']['dedicated'].append(device)
                elif 'vent.gpu.mem_mb' in container.attrs['Config']['Labels']:
                    if device not in gpu_status['vent_usage']['mem_mb']:
                        gpu_status['vent_usage']['mem_mb'][device] = 0
                    gpu_status['vent_usage']['mem_mb'][device] += int(
                        container.attrs['Config']['Labels']['vent.gpu.mem_mb'])
    except Exception as e:  # pragma: no cover
        logger.error('Could not get running jobs ' + str(e))

    port = '3476'
    # default docker gateway
    host = '172.17.0.1'
    result = template.option('nvidia-docker-plugin', 'port')
    if result[0]:
        port = result[1]
    result = template.option('nvidia-docker-plugin', 'host')
    if result[0]:
        host = result[1]
    else:
        try:
            # now just requires ip, ifconfig
            route = check_output(('ip', 'route')).decode('utf-8').split('\n')
            default = ''
            # grab the default network device.
            for device in route:
                if 'default' in device:
                    default = device.split()[4]
                    break

            # grab the IP address for the default device
            ip_addr = check_output(('ifconfig', default)).decode('utf-8')
            ip_addr = ip_addr.split('\n')[1].split()[1]
            host = ip_addr
        except Exception as e:  # pragma: no cover
            logger.error('Something with the ip addresses'
                         'went wrong ' + str(e))

    # have to get the info separately to determine how much memory is availabe
    nd_url = 'http://' + host + ':' + port + '/v1.0/gpu/info/json'
    try:
        r = requests.get(nd_url)
        if r.status_code == 200:
            status = r.json()
            for i, device in enumerate(status['Devices']):
                gm = int(round(math.log(int(device['Memory']['Global']), 2)))
                gpu_status[i] = {'global_memory': 2**gm,
                                 'cores': device['Cores']}
        else:
            usage = (False, 'Unable to get GPU usage request error code: ' +
                     str(r.status_code))
    except Exception as e:  # pragma: no cover
        usage = (False, 'Error: ' + str(e))

    # get actual status of each gpu
    nd_url = 'http://' + host + ':' + port + '/v1.0/gpu/status/json'
    try:
        r = requests.get(nd_url)
        if r.status_code == 200:
            status = r.json()
            for i, device in enumerate(status['Devices']):
                if i not in gpu_status:
                    gpu_status[i] = {}
                gpu_status[i]['utilization'] = device['Utilization']
                gpu_status[i]['memory'] = device['Memory']
                gpu_status[i]['processes'] = device['Processes']
            usage = (True, gpu_status)
        else:
            usage = (False, 'Unable to get GPU usage request error code: ' +
                     str(r.status_code))
    except Exception as e:  # pragma: no cover
        usage = (False, 'Error: ' + str(e))

    return usage
207092
def get_host_usage(self):
        
        import psutil
        process_names = [proc.name for proc in psutil.process_iter()]
        cpu_pct = psutil.cpu_percent(interval=1)
        mem = psutil.virtual_memory()
        return str(cpu_pct), str(len(process_names)), str(mem.available), str(mem.total)
559259
def __gather_avail(self):
        '''
        
        '''
        avail = {}
        for saltenv in self._get_envs():
            avail[saltenv] = self.client.list_states(saltenv)
        return avail
155590
def main(
        gpu:Param("GPU to run on", str)=None,
        lr: Param("Learning rate", float)=1e-3,
        size: Param("Size (px: 128,192,224)", int)=128,
        debias_mom: Param("Debias statistics", bool)=False,
        debias_sqr: Param("Debias statistics", bool)=False,
        opt: Param("Optimizer: 'adam','genopt','rms','sgd'", str)='genopt',
        alpha: Param("Alpha", float)=0.99,
        mom: Param("Momentum", float)=0.9,
        eps: Param("epsilon", float)=1e-7,
        decay: Param("Decay AvgStatistic (momentum)", bool)=False,
        epochs: Param("Number of epochs", int)=5,
        bs: Param("Batch size", int)=128,
        ):
    """"""

    # Pick one of these
    gpu = setup_distrib(gpu)
    if gpu is None: bs *= torch.cuda.device_count()

    moms = (mom,mom)
    stats = [
            AvgStatistic('momentum', mom,   scope=StatScope.Weight, decay=decay, debias=debias_mom),
            AvgSquare   ('alpha',    alpha, scope=StatScope.Weight, debias=debias_sqr),
            ConstStatistic('eps', eps), ConstStatistic('clip', 0.001),
            ]
    if   opt=='adam'  : opt_func = partial(optim.Adam, betas=(mom,alpha), eps=eps)
    elif opt=='rms'   : opt_func = partial(optim.RMSprop, alpha=alpha)
    elif opt=='genopt': opt_func = partial(GeneralOptimizer, on_step=on_step, stats=stats)
    else: raise Exception(f'unknown opt: {opt}')

    #opt_func = optim.SGD
    #learn = (cnn_learner(data, models.xresnet50, pretrained=False, concat_pool=False, lin_ftrs=[], split_on=bn_and_final,
    print(f'lr: {lr}; size: {size}; debias_mom: {debias_mom}; debias_sqr: {debias_sqr}; opt: {opt}; alpha: {alpha}; mom: {mom}; eps: {eps}; decay: {decay}')
    print('imagenette')
    get_learner(lr, size, False, bs, opt_func, gpu, epochs)
    gc.collect()

    print('imagewoof')
    get_learner(lr, size, True, bs, opt_func, gpu, epochs)
649470
def sg_gpus():
    
    """
    global _gpus

    if _gpus is None:
        local_device_protos = device_lib.list_local_devices()
        _gpus = len([x.name for x in local_device_protos if x.device_type == 'GPU'])

    return max(_gpus, 1)
353234
private RuntimeOptions getRuntimeOptions() {
    RuntimeOptions opts = new RuntimeOptions();
    opts.setSeed((int) get_params().getOrMakeRealSeed());
    opts.setUseGPU(get_params()._gpu);
    opts.setDeviceID(get_params()._device_id);
    return opts;
  }
765376
def collect_via_nvidia_smi(self, stats_config):
        
        raw_output = self.run_command([
            '--query-gpu={query_gpu}'.format(query_gpu=','.join(stats_config)),
            '--format=csv,nounits,noheader'
        ])

        if raw_output is None:
            return

        results = raw_output[0].strip().split("\n")
        for result in results:
            stats = result.strip().split(',')
            assert len(stats) == len(stats_config)
            index = stats[0]
            for stat_name, metric in izip(stats_config[1:], stats[1:]):
                metric_name = 'gpu_{index}.{stat_name}'.format(
                    index=str(index),
                    stat_name=stat_name
                )
                self.publish(metric_name, metric)
28346
def raw_nogpu_session(graph=None):
  """"""
  config = tf.compat.v1.ConfigProto(device_count={'GPU': 0})
  return tf.compat.v1.Session(config=config, graph=graph)
1306365
def device_by_load(wproc=0.5, wmem=0.5):
    

    gl = gpu_load(wproc=wproc, wmem=wmem)
    # return np.argsort(np.asarray(gl)[:, -1]).tolist()
    return [idx for idx, load in sorted(enumerate(
        [g.weighted for g in gl]), key=(lambda x: x[1]))]
350370
def usable_cpu_count():
    
    try:
        result = len(os.sched_getaffinity(0))
    except AttributeError:
        try:
            result = len(psutil.Process().cpu_affinity())
        except AttributeError:
            result = os.cpu_count()
    return result
23642
def main(
    gpus:Param("The GPUs to use for distributed training", str)='all',
    script:Param("Script to run", str, opt=False)='',
    args:Param("Args to pass to script", nargs='...', opt=False)=''
):
    ""
    # Loosely based on torch.distributed.launch
    current_env = os.environ.copy()
    gpus = list(range(torch.cuda.device_count())) if gpus=='all' else list(gpus)
    current_env["WORLD_SIZE"] = str(len(gpus))
    current_env["MASTER_ADDR"] = '127.0.0.1'
    current_env["MASTER_PORT"] = '29500'

    processes = []
    for i,gpu in enumerate(gpus):
        current_env["RANK"] = str(i)
        cmd = [sys.executable, "-u", script, f"--gpu={gpu}"] + args
        process = subprocess.Popen(cmd, env=current_env)
        processes.append(process)

    for process in processes: process.wait()
826935
def usage(self):
        
        if self._usage is None:
            self._usage = UsageList(self._version, account_sid=self._solution['sid'], )
        return self._usage
930186
def gpu_mem_trace(func):
    ""
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        with GPUMemTrace(ctx=func.__qualname__, on_exit_report=True):
            return func(*args, **kwargs)
    return wrapper
140137
def get_cuda_visible_devices():
    
    gpu_ids_str = os.environ.get("CUDA_VISIBLE_DEVICES", None)

    if gpu_ids_str is None:
        return None

    if gpu_ids_str == "":
        return []

    return [int(i) for i in gpu_ids_str.split(",")]
1102304
def requiredGPU_MB(self, n):
        
        from darknet.core import darknet_with_cuda
        if (darknet_with_cuda()): # its using cuda
            free = getFreeGPU_MB()
            print("Yolo: requiredGPU_MB: required, free", n, free)
            if (free == -1): # could not detect ..
                return True
            return (free>=n)
        else:
            return True
239564
def setting_ctx(num_gpus):
    
    if num_gpus > 0:
        ctx = [mx.gpu(i) for i in range(num_gpus)]
    else:
        ctx = [mx.cpu()]
    return ctx
768368
def ncores_used(self):
        
        return sum(task.manager.num_cores for task in self if task.status == task.S_RUN)