ORM teardown sequencing for all replica set pools
175957
function () {
        // release all pending connections
        util.each(this.connections, function (connection, threadId, connections) {
            try {
                connection.release();
            }
            catch (e) { } // nothing to do with error
            delete connections[threadId];
        });

        // execute end on the db. will end pool if pool, or otherwise will execute whatever `end` that has been
        // exposed by db.js
        util.each(this.sources, function (source, identity, sources) {
            try {
                source && source.end();
            }
            catch (e) { } // nothing to do with error
            delete sources[identity];
        });
    }
616044
private   final void dcompute() {// Work to do the distribution
    // Split out the keys into disjointly-homed sets of keys.
    // Find the split point.  First find the range of home-indices.
    H2O cloud = H2O.CLOUD;
    int lo=cloud._memary.length, hi=-1;
    for( Key k : _keys ) {
      int i = k.home(cloud);
      if( i<lo ) lo=i;
      if( i>hi ) hi=i;        // lo <= home(keys) <= hi
    }

    // Classic fork/join, but on CPUs.
    // Split into 3 arrays of keys: lo keys, hi keys and self keys
    final ArrayList<Key> locals = new ArrayList<Key>();
    final ArrayList<Key> lokeys = new ArrayList<Key>();
    final ArrayList<Key> hikeys = new ArrayList<Key>();
    int self_idx = cloud.nidx(H2O.SELF);
    int mid = (lo+hi)>>>1;    // Mid-point
    for( Key k : _keys ) {
      int idx = k.home(cloud);
      if( idx == self_idx ) locals.add(k);
      else if( idx < mid )  lokeys.add(k);
      else                  hikeys.add(k);
    }

    // Launch off 2 tasks for the other sets of keys, and get a place-holder
    // for results to block on.
    _lo = remote_compute(lokeys);
    _hi = remote_compute(hikeys);

    // Setup for local recursion: just use the local keys.
    if( locals.size() != 0 ) {  // Shortcut for no local work
      _local = clone();         // 'this' is completer for '_local', so awaits _local completion
      _local._is_local = true;
      _local._keys = locals.toArray(new Key[locals.size()]); // Keys, including local keys (if any)
      _local.init();            // One-time top-level init
      H2O.submitTask(_local);   // Begin normal execution on a FJ thread
    } else {
      tryComplete();            // No local work, so just immediate tryComplete
    }
  }
224556
private void tearDown() throws Exception {
    try {
      if (fileSystem != null) {
        fileSystem.delete(new Path(ROOT), true, true);
      }
    } finally {
      RPC.stopProxy(proxyClientProxyProtocol);
      IOUtils.cleanup(LOG, proxyTClientProxyProtocol, clientManager, fileSystem);
    }
  }
1030220
def Cleanup(self):
		
		self.lock.acquire()
		try:
			for bucket in self.connections.values():
				try:
					for conn in bucket:
						conn.lock()
						try:
							open = conn.TestConnection(forceCheck=True)
							if open is True:
								conn.commit()
							else:
								#Remove the connection from the pool. Its dead better of recreating it.
								index = bucket.index(conn)
								del bucket[index]
							conn.release()
						except Exception:
							conn.release()
				except Exception:
					pass
		finally:
			self.lock.release()
1331805
def load_orthologs(fo: IO, metadata: dict):
    

    version = metadata["metadata"]["version"]

    # LOAD ORTHOLOGS INTO ArangoDB
    with timy.Timer("Load Orthologs") as timer:
        arango_client = arangodb.get_client()
        belns_db = arangodb.get_belns_handle(arango_client)
        arangodb.batch_load_docs(
            belns_db, orthologs_iterator(fo, version), on_duplicate="update"
        )

        log.info(
            "Load orthologs",
            elapsed=timer.elapsed,
            source=metadata["metadata"]["source"],
        )

        # Clean up old entries
        remove_old_ortholog_edges = f"""
            FOR edge in ortholog_edges
                FILTER edge.source == "{metadata["metadata"]["source"]}"
                FILTER edge.version != "{version}"
                REMOVE edge IN ortholog_edges
        """
        remove_old_ortholog_nodes = f"""
            FOR node in ortholog_nodes
                FILTER node.source == "{metadata["metadata"]["source"]}"
                FILTER node.version != "{version}"
                REMOVE node IN ortholog_nodes
        """
        arangodb.aql_query(belns_db, remove_old_ortholog_edges)
        arangodb.aql_query(belns_db, remove_old_ortholog_nodes)

    # Add metadata to resource metadata collection
    metadata["_key"] = f"Orthologs_{metadata['metadata']['source']}"
    try:
        belns_db.collection(arangodb.belns_metadata_name).insert(metadata)
    except ArangoError as ae:
        belns_db.collection(arangodb.belns_metadata_name).replace(metadata)
1060718
def _find_families_deprecated(clus_obj, min_seqs):
    
    logger.info("Creating meta-clusters based on shared sequences.")
    seen = defaultdict()
    metacluster = defaultdict(list)
    c_index = clus_obj.keys()
    meta_idx = 0
    with ProgressBar(maxval=len(c_index), redirect_stdout=True) as p:
        for itern, c in enumerate(c_index):
            p.update(itern)
            clus = clus_obj[c]
            if len(clus.idmembers.keys()) < min_seqs:
                del clus_obj[c]
                continue
            logger.debug("reading cluster %s" % c)
            logger.debug("loci2seq  %s" % clus.loci2seq)
            already_in, not_in = _get_seqs_from_cluster(clus.idmembers.keys(), seen)
            logger.debug("seen %s news %s" % (already_in, not_in))
            meta_idx += 1
            metacluster[meta_idx].append(c)
            seen.update(dict(zip(not_in, [meta_idx] * len(not_in))))
            if len(already_in) > 0:
                logger.debug("seen in %s" % already_in)
                for eindex in already_in:
                    for cluster in metacluster[eindex]:
                        metacluster[meta_idx].append(cluster)
                        prev_clus = clus_obj[cluster]
                        logger.debug("_find_families: prev %s current %s" % (eindex, clus.id))
                        # add current seqs to seen cluster
                        seqs_in = prev_clus.idmembers.keys()
                        seen.update(dict(zip(seqs_in, [meta_idx] * len(seqs_in))))
                        # for s_in_clus in prev_clus.idmembers:
                        #    seen[s_in_clus] = meta_idx
                    #    clus.idmembers[s_in_clus] = 1
                    # add current locus to seen cluster
                    # for loci in prev_clus.loci2seq:
                    #    logger.debug("adding %s" % loci)
                        # if not loci_old in current_clus[eindex].loci2seq:
                    #    clus.add_id_member(list(prev_clus.loci2seq[loci]), loci)
                    # logger.debug("loci %s" % clus.loci2seq.keys())
                    del metacluster[eindex]
                # clus_obj[c] = clus

                # logger.debug("num cluster %s" % len(clus_obj.keys()))
    logger.info("%s clusters merged" % len(metacluster))

    return metacluster, seen
68819
@Override
  @TearDown(Level.Trial)
  public void teardown() throws Exception {
    completed.set(true);
    Thread.sleep(5000);
    super.teardown();
  }
953575
def _repopulate_pool(self):
        
        for i in range(self._processes - len(self._pool)):
            w = self.Process(target=worker,
                             args=(self._inqueue, self._outqueue,
                                   self._initializer,
                                   self._initargs, self._maxtasksperchild,
                                   self._wrap_exception,
                                   self._finalizer,
                                   self._finalargs)
                             )
            self._pool.append(w)
            w.name = w.name.replace('Process', 'PoolWorker')
            w.daemon = True
            w.start()
            util.debug('added worker')
1231458
def _prepare_output(partitions, verbose):
    """"""
    out = {}
    partitions_count = len(partitions)
    out['raw'] = {
        'not_enough_replicas_count': partitions_count,
    }

    if partitions_count == 0:
        out['message'] = 'All replicas in sync.'
    else:
        out['message'] = (
            "{0} partition(s) have the number of replicas in "
            "sync that is lower than the specified min ISR."
        ).format(partitions_count)

        if verbose:
            lines = (
                "isr={isr} is lower than min_isr={min_isr} for {topic}:{partition}"
                .format(
                    isr=p['isr'],
                    min_isr=p['min_isr'],
                    topic=p['topic'],
                    partition=p['partition'],
                )
                for p in partitions
            )
            out['verbose'] = "Partitions:\n" + "\n".join(lines)
    if verbose:
        out['raw']['partitions'] = partitions

    return out
79123
def teardown
      raise "Only Ec2 teardown supported" unless cloud_provider.name.to_s == 'ec2'
      puts "! Tearing down cloud #{name}"
      # load_balancers.each do |name, lb|
      #   puts "! Deleting load_balaner #{lb_name}"
      #   lb.teardown
      # end
      load_balancers.each do |lb|
        puts "-----> Tearing down load balancer: #{lb.name}"
        lb.teardown
      end

      rds_instances.each do |rds|
        puts "-----> Tearing down RDS Instance: #{rds.name}"
        rds.teardown
      end
      # instances belonging to an auto_scaling group must be deleted before the auto_scaling group
      #THIS SCARES ME! nodes.each{|n| n.terminate_instance!}
      # loop {nodes.size>0 ? sleep(4) : break }
      if autoscalers.empty?
        nodes.each do |node|
          node.terminate!
        end
      else
        autoscalers.each do |a|
          puts "-----> Tearing down autoscaler #{a.name}"
          a.teardown
        end
      end
      # autoscalers.keys.each do |as_name|
      #   puts "! Deleting auto_scaling_group #{as_name}"
      #   cloud_provider.as.delete_autoscaling_group('AutoScalingGroupName' => as_name)
      # end
      #TODO: keypair.delete # Do we want to delete the keypair?  probably, but not certain
    end
49951
def DeleteRecords(cls, ids, token):
    
    with data_store.DB.GetMutationPool() as mutation_pool:
      mutation_pool.QueueDeleteRecords(ids)
1369137
def update(self):
        
        ring = self._fetch()
        n_replicas = len(ring)
        replica_set = set([r[1] for r in self.replicas])
        self.ranges = []
        for n, (start, replica) in enumerate(ring):
            if replica in replica_set:
                end = ring[(n+1) % n_replicas][0] % RING_SIZE
                if start < end:
                    self.ranges.append((start, end))
                elif end < start:
                    self.ranges.append((start, RING_SIZE))
                    self.ranges.append((0, end))
                else:
                    self.ranges.append((0, RING_SIZE))
729676
def teardown_databases(self, old_config, options):
        
        if len(old_config) > 1:
            old_names, mirrors = old_config
        else:
            old_names = old_config
        for connection, old_name, destroy in old_names:
            if destroy:
                connection.creation.destroy_test_db(old_name, options['verbosity'])
472260
private void removeBlocks(List<BlockInfo> blocks) {
    if (blocks == null) {
      return;
    }
    for (BlockInfo b : blocks) {
      removeFromExcessReplicateMap(b);
      neededReplications.remove(b, -1);
      corruptReplicas.removeFromCorruptReplicasMap(b);
      if (pendingReplications != null) {
        int replicas = pendingReplications.getNumReplicas(b);
        for (int i = 0; i < replicas; i++) {
          pendingReplications.remove(b);
        }    
      }    
      addToInvalidates(b, false);
      blocksMap.removeBlock(b);
    }
  }
822601
void clearReplicationQueues() {
    writeLock();
    try {
      synchronized (neededReplications) {
        neededReplications.clear();
      }
      underReplicatedBlocksCount = 0;

      corruptReplicas.clear();
      corruptReplicaBlocksCount = 0;

      overReplicatedBlocks.clear();
      raidEncodingTasks.clear();

      excessReplicateMap = new HashMap<String, LightWeightHashSet<Block>>();
      excessBlocksCount = 0;
    } finally {
      writeUnlock();
    }
  }
249907
def after_delete_oai_set(mapper, connection, target):
    """"""
    _delete_percolator(spec=target.spec, search_pattern=target.search_pattern)
    sleep(2)
    update_affected_records.delay(
        spec=target.spec
    )
676893
def clear_synonyms!(forward_to_replicas = false, request_options = {})
      res = clear_synonyms(forward_to_replicas, request_options)
      wait_task(res['taskID'], WAIT_TASK_DEFAULT_TIME_BEFORE_RETRY, request_options)
      res
    end