Prepares gemini databases in parallel, handling jointly called populations.
352669
def prep_db_parallel(samples, parallel_fn):
    
    batch_groups, singles, out_retrieve, extras = _group_by_batches(samples, _has_variant_calls)
    to_process = []
    has_batches = False
    for (name, caller), info in batch_groups.items():
        fnames = [x[0] for x in info]
        to_process.append([fnames, (str(name), caller, True), [x[1] for x in info], extras])
        has_batches = True
    for name, caller, data, fname in singles:
        to_process.append([[fname], (str(name), caller, False), [data], extras])
    output = parallel_fn("prep_gemini_db", to_process)
    out_fetch = {}
    for batch_id, out_file in output:
        out_fetch[tuple(batch_id)] = out_file
    out = []
    for batch_name, data in out_retrieve:
        out_variants = []
        for vrn in data["variants"]:
            use_population = vrn.pop("population", True)
            if use_population:
                vrn["population"] = out_fetch[(batch_name, vrn["variantcaller"])]
            out_variants.append(vrn)
        data["variants"] = out_variants
        out.append([data])
    for x in extras:
        out.append([x])
    return out
1062193
def delayed_bamprep_merge(samples, run_parallel):
    
    if any("combine" in data[0] for data in samples):
        return run_parallel("delayed_bam_merge", samples)
    else:
        return samples
556014
private void processNeighbors(DoubleDBIDListIter neighbor, ModifiableDBIDs currentCluster, ArrayModifiableDBIDs seeds) {
    final boolean ismetric = getDistanceFunction().isMetric();
    for(; neighbor.valid(); neighbor.advance()) {
      if(processedIDs.add(neighbor)) {
        if(!ismetric || neighbor.doubleValue() > 0.) {
          seeds.add(neighbor);
        }
      }
      else if(!noise.remove(neighbor)) {
        continue;
      }
      currentCluster.add(neighbor);
    }
  }
511859
def run(bam_file, data, out_dir):
    """"""
    m = {"base": None, "secondary": []}
    m.update(_mirbase_stats(data, out_dir))
    m["secondary"].append(_seqcluster_stats(data, out_dir))
569259
def data_parallelism(daisy_chain_variables=True,
                     all_workers=False,
                     ps_replicas=0,
                     ps_job="/job:ps",
                     ps_gpu=0,
                     schedule="continuous_train_and_eval",
                     sync=False,
                     worker_gpu=1,
                     worker_replicas=1,
                     worker_id=0,
                     gpu_order="",
                     worker_job="/job:localhost",
                     no_data_parallelism=False):
  """"""
  tf.logging.info("schedule=%s" % schedule)
  tf.logging.info("worker_gpu=%s" % worker_gpu)
  tf.logging.info("sync=%s" % sync)
  def _ps_replicas(all_workers=False):
    if all_workers:
      return list(range(ps_replicas))
    # Worker K will be using replicas {0,...n-1} + K*n if we have n replicas.
    num_replicas = ps_replicas // worker_replicas
    return [d + worker_id * num_replicas for d in range(num_replicas)]

  def _gpu_order(num_gpus):
    if gpu_order:
      ret = [int(s) for s in gpu_order.split(" ")]
      if len(ret) == num_gpus:
        return ret
    return list(range(num_gpus))

  def _ps_gpus(all_workers=False):
    ps_gpus = []
    for d in _ps_replicas(all_workers=all_workers):
      ps_gpus.extend([(d, gpu) for gpu in _gpu_order(ps_gpu)])
    return ps_gpus

  def ps_devices(all_workers=False):
    """List of ps devices (where to put the experts).

    Args:
      all_workers: whether the list is for all async workers or just this one.

    Returns:
      a list of device names
    """
    if ps_replicas > 0:
      if ps_gpu > 0:
        return [
            ps_job + "/task:%d/GPU:%d" % (d, gpu)
            for (d, gpu) in _ps_gpus(all_workers=all_workers)
        ]
      else:
        return [
            ps_job + "/task:%d" % d
            for d in _ps_replicas(all_workers=all_workers)
        ]
    else:
      if worker_gpu > 0:
        return ["gpu:%d" % d for d in _gpu_order(worker_gpu)]
      else:
        return [""]

  def _replica_device_setter(worker_device):
    if ps_replicas == 0:
      return worker_device
    return tf.train.replica_device_setter(
        worker_device=worker_device,
        ps_tasks=ps_replicas,
        ps_device=ps_job + "/GPU:0" if ps_gpu > 0 else ps_job)

  is_single_machine = ps_replicas == 0 and worker_replicas == 1

  if no_data_parallelism:
    datashard_devices = [""]
    caching_devices = None
  elif is_single_machine:
    tf.logging.warn(
        "Schedule=%s. Assuming that training is running on a single machine.",
        schedule)
    datashard_devices = ["gpu:%d" % d for d in _gpu_order(worker_gpu)]
    if worker_gpu < 1:
      datashard_devices += ["cpu:0"]
    caching_devices = None
  elif sync and ps_replicas > 0:
    # compute on ps
    datashard_devices = [
        _replica_device_setter(d) for d in ps_devices(all_workers=all_workers)
    ]
    if ps_gpu > 0 and ps_replicas > 1:
      caching_devices = [
          ps_job + "/task:%d/cpu:0" % d
          for (d, _) in _ps_gpus(all_workers=all_workers)
      ]
    else:
      caching_devices = None
  else:
    # compute on worker - this is either a single-worker setup or asynchronous
    # with parameter servers.
    if worker_gpu > 1:
      datashard_devices = [
          _replica_device_setter(worker_job + "/GPU:%d" % d)
          for d in _gpu_order(worker_gpu)
      ]
      caching_devices = None
    else:
      datashard_devices = [_replica_device_setter(worker_job)]
      caching_devices = None
  tf.logging.info("datashard_devices: %s", datashard_devices)
  tf.logging.info("caching_devices: %s", caching_devices)
  tf.logging.info("ps_devices: %s", ps_devices(all_workers=all_workers))
  return eu.Parallelism(
      datashard_devices,
      caching_devices=caching_devices,
      daisy_chain_variables=daisy_chain_variables,
      ps_devices=ps_devices(all_workers=all_workers))
841751
def download_and_calibrate_parallel(list_of_ids, n=None):
    
    setup_cluster(n_cores=n)
    c = Client()
    lbview = c.load_balanced_view()
    lbview.map_async(download_and_calibrate, list_of_ids)
    subprocess.Popen(["ipcluster", "stop", "--quiet"])
1033529
public PCAResult processQueryResult(DoubleDBIDList results, Relation<? extends NumberVector> database) {
    return processCovarMatrix(covarianceMatrixBuilder.processQueryResults(results, database));
  }
271458
def prepare(
            self, engine=None, mode=None, model=None, index=None, key=None,
            filter=None, projection=None, consistent=None, forward=None, parallel=None):
        """"""

        self.prepare_iterator_cls(engine, mode)
        self.prepare_model(model, index, consistent)
        self.prepare_key(key)
        self.prepare_projection(projection)
        self.prepare_filter(filter)
        self.prepare_constraints(forward, parallel)

        self.prepare_request()
1099862
def _distribute_frame_compute(self, writers, frame_keys, frames_to_write, client, batch_size=1):
        """"""
        def load_data(frame_gen, q):
            for frame_arrays in frame_gen:
                future_list = client.compute(frame_arrays)
                for frame_key, arr_future in zip(frame_keys, future_list):
                    q.put({frame_key: arr_future})
            q.put(None)

        input_q = Queue(batch_size if batch_size is not None else 1)
        load_thread = Thread(target=load_data, args=(frames_to_write, input_q,))
        remote_q = client.gather(input_q)
        load_thread.start()

        while True:
            future_dict = remote_q.get()
            if future_dict is None:
                break

            # write the current frame
            # this should only be one element in the dictionary, but this is
            # also the easiest way to get access to the data
            for frame_key, result in future_dict.items():
                # frame_key = rev_future_dict[future]
                w = writers[frame_key]
                w.append_data(result)
            input_q.task_done()

        log.debug("Waiting for child thread...")
        load_thread.join(10)
        if load_thread.is_alive():
            import warnings
            warnings.warn("Background thread still alive after failing to die gracefully")
        else:
            log.debug("Child thread died successfully")
984831
public void process(List<EllipseInfo> ellipses , List<List<Node>> output ) {

		init(ellipses);

		connect(ellipses);

		output.clear();
		for (int i = 0; i < clusters.size(); i++) {
			List<Node> c = clusters.get(i);

			// remove noise
			removeSingleConnections(c);

			if( c.size() >= minimumClusterSize) {
				output.add(c);
			}
		}
	}
488221
def process_raw_data(cls, raw_data):
        """"""
        properties = raw_data.get("properties", {})

        load_balancing_rules = []
        for raw_content in properties.get("loadBalancingRules", []):
            resource = Resource.from_raw_data(raw_content)
            load_balancing_rules.append(resource)
        properties["loadBalancingRules"] = load_balancing_rules

        return super(Probes, cls).process_raw_data(raw_data)
948581
def _prepare_output(partitions, unavailable_brokers, verbose):
    """"""
    partitions_count = len(partitions)
    out = {}
    out['raw'] = {
        'replica_unavailability_count': partitions_count,
    }

    if partitions_count == 0:
        out['message'] = 'All replicas available for communication.'
    else:
        out['message'] = "{replica_unavailability} replicas unavailable for communication. " \
            "Unavailable Brokers: {unavailable_brokers}".format(
            replica_unavailability=partitions_count,
            unavailable_brokers=', '.join([str(e) for e in unavailable_brokers]),
        )
        if verbose:
            lines = (
                '{}:{}'.format(topic, partition)
                for (topic, partition) in partitions
            )
            out['verbose'] = "Partitions:\n" + "\n".join(lines)

    if verbose:
        out['raw']['partitions'] = [
            {'topic': topic, 'partition': partition}
            for (topic, partition) in partitions
        ]

    return out
965979
def create_gemini_db(gemini_vcf, data, gemini_db=None, ped_file=None):
    
    if not gemini_db:
        gemini_db = "%s.db" % utils.splitext_plus(gemini_vcf)[0]
    if not vcfutils.vcf_has_variants(gemini_vcf):
        return None
    if not utils.file_exists(gemini_db):
        with file_transaction(data, gemini_db) as tx_gemini_db:
            vcf2db = config_utils.get_program("vcf2db.py", data)
            if "vcf2db_expand" in dd.get_tools_on(data):
                vcf2db_args = ["--expand", "gt_types", "--expand", "gt_ref_depths", "--expand", "gt_alt_depths"]
            else:
                vcf2db_args = []
            cmd = [vcf2db, gemini_vcf, ped_file, tx_gemini_db] + vcf2db_args
            do.run(cmd, "GEMINI: create database with vcf2db")
    return gemini_db
371941
def load(ctx, variant_source, family_file, family_type, root):
    
    root = root or ctx.obj.get('root') or os.path.expanduser("~/.puzzle")

    if os.path.isfile(root):
        logger.error("'root' can't be a file")
        ctx.abort()

    logger.info("Root directory is: {}".format(root))

    db_path = os.path.join(root, 'puzzle_db.sqlite3')
    logger.info("db path is: {}".format(db_path))
    
    if not os.path.exists(db_path):
        logger.warn("database not initialized, run 'puzzle init'")
        ctx.abort()


    if not os.path.isfile(variant_source):
        logger.error("Variant source has to be a file")
        ctx.abort()
    
    mode = get_file_type(variant_source)
    if mode == 'unknown':
        logger.error("Unknown file type")
        ctx.abort()
    #Test if gemini is installed
    elif mode == 'gemini':
        logger.debug("Initialzing GEMINI plugin")
        if not GEMINI:
            logger.error("Need to have gemini installed to use gemini plugin")
            ctx.abort()

    logger.debug('Set puzzle backend to {0}'.format(mode))
    
    variant_type = get_variant_type(variant_source)
    logger.debug('Set variant type to {0}'.format(variant_type))
    
    cases = get_cases(
        variant_source=variant_source,
        case_lines=family_file, 
        case_type=family_type, 
        variant_type=variant_type, 
        variant_mode=mode
    )
    
    if len(cases) == 0:
        logger.warning("No cases found")
        ctx.abort()

    logger.info("Initializing sqlite plugin")
    store = SqlStore(db_path)

    for case_obj in cases:
        if store.case(case_obj.case_id) is not None:
            logger.warn("{} already exists in the database"
                        .format(case_obj.case_id))
            continue
        # extract case information
        logger.debug("adding case: {} to puzzle db".format(case_obj.case_id))
        store.add_case(case_obj, vtype=variant_type, mode=mode)
928679
public static void alignDatanodes(DatanodeInfo[] dstLocs,
      DatanodeInfo[] srcLocs) {
    for (int i = 0; i < dstLocs.length; i++) {
      for (int j = 0; j < srcLocs.length; j++) {
        if (i == j)
          continue;
        if (dstLocs[i].equals(srcLocs[j])) {
          if (i < j) {
            swap(i, j, srcLocs);
          } else {
            swap(i, j, dstLocs);
          }
          break;
        }
      }
    }
  }
958512
def prepare_queues(queues, lock):
    """"""
    for queue in queues:
        queue._pebble_lock = lock
        with queue.mutex:
            queue._pebble_old_method = queue._put
            queue._put = MethodType(new_method, queue)
183650
public Clustering<CorrelationModel> run(Database database, Relation<V> relation) {
    final int dim = RelationUtil.dimensionality(relation);

    StepProgress stepprog = LOG.isVerbose() ? new StepProgress(3) : null;

    // Run Generalized DBSCAN
    LOG.beginStep(stepprog, 1, "Preprocessing local correlation dimensionalities and partitioning data");
    // FIXME: how to ensure we are running on the same relation?
    ERiCNeighborPredicate<V>.Instance npred = new ERiCNeighborPredicate<V>(settings).instantiate(database, relation);
    CorePredicate.Instance<DBIDs> cpred = new MinPtsCorePredicate(settings.minpts).instantiate(database);
    Clustering<Model> copacResult = new GeneralizedDBSCAN.Instance<>(npred, cpred, false).run();

    // extract correlation clusters
    LOG.beginStep(stepprog, 2, "Extract correlation clusters");
    List<List<Cluster<CorrelationModel>>> clusterMap = extractCorrelationClusters(copacResult, relation, dim, npred);
    if(LOG.isDebugging()) {
      StringBuilder msg = new StringBuilder("Step 2: Extract correlation clusters...");
      for(int corrDim = 0; corrDim < clusterMap.size(); corrDim++) {
        List<Cluster<CorrelationModel>> correlationClusters = clusterMap.get(corrDim);
        msg.append("\n\ncorrDim ").append(corrDim);
        for(Cluster<CorrelationModel> cluster : correlationClusters) {
          msg.append("\n  cluster ").append(cluster).append(", ids: ").append(cluster.getIDs().size());
          // .append(", level: ").append(cluster.getLevel()).append(", index:
          // ").append(cluster.getLevelIndex());
          // msg.append("\n basis " +
          // cluster.getPCA().getWeakEigenvectors().toString(" ", NF) +
          // " ids " + cluster.getIDs().size());
        }
      }
      LOG.debugFine(msg.toString());
    }
    if(LOG.isVerbose()) {
      int clusters = 0;
      for(List<Cluster<CorrelationModel>> correlationClusters : clusterMap) {
        clusters += correlationClusters.size();
      }
      LOG.verbose(clusters + " clusters extracted.");
    }

    // build hierarchy
    LOG.beginStep(stepprog, 3, "Building hierarchy");
    Clustering<CorrelationModel> clustering = new Clustering<>("ERiC clustering", "eric-clustering");
    buildHierarchy(clustering, clusterMap, npred);
    if(LOG.isDebugging()) {
      StringBuilder msg = new StringBuilder("Step 3: Build hierarchy");
      for(int corrDim = 0; corrDim < clusterMap.size(); corrDim++) {
        List<Cluster<CorrelationModel>> correlationClusters = clusterMap.get(corrDim);
        for(Cluster<CorrelationModel> cluster : correlationClusters) {
          msg.append("\n  cluster ").append(cluster).append(", ids: ").append(cluster.getIDs().size());
          // .append(", level: ").append(cluster.getLevel()).append(", index:
          // ").append(cluster.getLevelIndex());
          for(It<Cluster<CorrelationModel>> iter = clustering.getClusterHierarchy().iterParents(cluster); iter.valid(); iter.advance()) {
            msg.append("\n   parent ").append(iter.get());
          }
          for(It<Cluster<CorrelationModel>> iter = clustering.getClusterHierarchy().iterChildren(cluster); iter.valid(); iter.advance()) {
            msg.append("\n   child ").append(iter.get());
          }
        }
      }
      LOG.debugFine(msg.toString());
    }
    LOG.setCompleted(stepprog);

    for(Cluster<CorrelationModel> rc : clusterMap.get(clusterMap.size() - 1)) {
      clustering.addToplevelCluster(rc);
    }
    return clustering;
  }