Run the deduplication process.

        We apply the removal strategy one duplicate set at a time to keep
        memory footprint low and make the log of actions easier to read.
19666
def run(self):
        
        logger.info(
            "The {} strategy will be applied on each duplicate set.".format(
                self.conf.strategy))

        self.stats['set_total'] = len(self.mails)

        for hash_key, mail_path_set in self.mails.items():
            # Print visual clue to separate duplicate sets.
            logger.info('---')

            duplicates = DuplicateSet(hash_key, mail_path_set, self.conf)

            # Perfom the deduplication.
            duplicates.dedupe()

            # Merge stats resulting of actions on duplicate sets.
            self.stats += duplicates.stats
525345
def finalize(self):
        """"""
        if self.total_duplicate > 0:
            print('{} duplicate scripts found'.format(self.total_duplicate))
            for duplicate in self.list_duplicate:
                print(duplicate)
875551
def _is_duplicate_record(self, rtype, name, content):
        """"""
        records = self._list_records(rtype, name, content)
        is_duplicate = len(records) >= 1
        if is_duplicate:
            LOGGER.info('Duplicate record %s %s %s, NOOP', rtype, name, content)
        return is_duplicate
593269
protected void retryReplicationIfError() {
        Log.d(TAG, "retryReplicationIfError() state=" + stateMachine.getState() +
                ", error=" + this.error +
                ", isContinuous()=" + isContinuous() +
                ", isTransientError()=" + Utils.isTransientError(this.error));

        // Make sure if state is IDLE, this method should be called when state becomes IDLE
        if (!stateMachine.getState().equals(ReplicationState.IDLE))
            return;

        if (this.error != null) {
            // IDLE_ERROR
            if (isContinuous()) {
                // 12/16/2014 - only retry if error is transient error 50x http error
                // It may need to retry for any kind of errors
                if (Utils.isTransientError(this.error)) {
                    onBeforeScheduleRetry();
                    cancelRetryFuture();
                    scheduleRetryFuture();
                }
            }
        }
    }
1355445
def duplicate(self):
        '''
        
        '''
        return self.__class__(amount=self.amount, date=self.date,
                              method=self.method, ref=self.ref)
610602
def _rollback(self):
        

        # to avoid using self set that may be updated during the process
        indexed_values = set(self._indexed_values)
        deindexed_values = set(self._deindexed_values)

        for args in indexed_values:
            self.remove(*args)
        for args in deindexed_values:
            self.add(*args, check_uniqueness=False)
445664
private static synchronized void decrementUseCount() {
        final String methodName = "decrementUseCount";
        logger.entry(methodName);

        --useCount;
        if (useCount <= 0) {
            if (bootstrap != null) {
              bootstrap.group().shutdownGracefully(0, 500, TimeUnit.MILLISECONDS);
            }
            bootstrap = null;
            useCount = 0;
        }

        logger.exit(methodName);
    }
9516
private void checkDuplicate() {
		KeyboardShortcut ks = this.getDuplicate();
		if (ks != null) {
			this.setFieldValue(FIELD_INFO, Constant.messages.getString("keyboard.dialog.warning.dup", ks.getName()));
		} else {
			this.setFieldValue(FIELD_INFO, "");
		}
	}
1292257
def _detect_duplicates(bam_file, out_dir, data):
    
    out_file = os.path.join(out_dir, "dup_metrics.txt")
    if not utils.file_exists(out_file):
        dup_align_bam = postalign.dedup_bam(bam_file, data)
        logger.info("Detecting duplicates in %s." % dup_align_bam)
        dup_count = readstats.number_of_mapped_reads(data, dup_align_bam, keep_dups=False)
        tot_count = readstats.number_of_mapped_reads(data, dup_align_bam, keep_dups=True)
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                out_handle.write("%s\n%s\n" % (dup_count, tot_count))
    with open(out_file) as in_handle:
        dupes = float(next(in_handle).strip())
        total = float(next(in_handle).strip())
    if total == 0:
        rate = "NA"
    else:
        rate = dupes / total
    return {"Duplication Rate of Mapped": rate}
1303382
public DuplicationReport monitorDuplication() {
        log.info("starting duplication monitor");
        DuplicationReport report = new DuplicationReport();

        for (String host : dupHosts.keySet()) {
            DuplicationInfo info = new DuplicationInfo(host);
            try {
                // Connect to storage providers
                ContentStoreManager storeManager = getStoreManager(host);
                ContentStore primary = storeManager.getPrimaryContentStore();
                String primaryStoreId = primary.getStoreId();
                List<ContentStore> secondaryList =
                    getSecondaryStores(storeManager, primaryStoreId);

                // Get primary space listing and count
                List<String> primarySpaces = getSpaces(host, primary);
                countSpaces(host, info, primary, primarySpaces, true);

                // Get space listing and space counts for secondary providers
                for (ContentStore secondary : secondaryList) {
                    List<String> secondarySpaces = getSpaces(host, secondary);
                    if (primarySpaces.size() != secondarySpaces.size()) {
                        info.addIssue("The spaces listings do not match " +
                                      "between primary and secondary " +
                                      "provider: " +
                                      secondary.getStorageProviderType());
                    }
                    // Determine item count for secondary provider spaces
                    countSpaces(host, info, secondary, secondarySpaces, false);
                }

                // Compare the space counts between providers
                compareSpaces(primaryStoreId, info);
            } catch (Exception e) {
                String error = e.getClass() + " exception encountered while " +
                               "running dup monitor for host " + host +
                               ". Exception message: " + e.getMessage();
                log.error(error);
                info.addIssue(error);
            } finally {
                report.addDupInfo(host, info);
            }
        }

        return report;
    }
953415
void processPendingReplications() {
    BlockInfo[] timedOutItems = pendingReplications.getTimedOutBlocks();
    if (timedOutItems != null) {
      writeLock();
      try {
        for (int i = 0; i < timedOutItems.length; i++) {
          NumberReplicas num = countNodes(timedOutItems[i]);
          neededReplications.add(
            timedOutItems[i], 
            num.liveReplicas(),
            num.decommissionedReplicas(),
            getReplication(timedOutItems[i]));
        }
      } finally {
        writeUnlock();
      }
      /* If we know the target datanodes where the replication timedout,
       * we could invoke decBlocksScheduled() on it. Its ok for now.
       */
    }
  }
1088397
private void processOverReplicatedBlock(Block block, short replication,
                                          DatanodeDescriptor addedNode, DatanodeDescriptor delNodeHint) {

    List<DatanodeID> excessReplicateTmp = new ArrayList<DatanodeID>();
    List<DatanodeID> originalDatanodes = new ArrayList<DatanodeID>();

    // find all replicas that can possibly be deleted.
    // The results are returned in excessReplicateTmp.
    findOverReplicatedReplicas(block, replication,
      addedNode, delNodeHint,
      excessReplicateTmp, originalDatanodes);
    if (excessReplicateTmp.size() <= 0) {
      return;
    }

    writeLock();                     // acquire write lock,
    try {
      BlockInfo storedBlock = blocksMap.getBlockInfo(block);
      INodeFile inode = (storedBlock == null) ? null : storedBlock.getINode(); 
      if (inode == null) {
        return; // file has been deleted already, nothing to do.
      }

      //
      // if the state of replicas of this block has changed since the time
      // when we released and reacquired the lock, then all the decisions
      // that we have made so far might not be correct. Do not delete excess
      // replicas in this case.

      int live = 0;
      Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block);
      for (Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block);
           it.hasNext();) {
        DatanodeDescriptor node = it.next();
        if (((nodesCorrupt != null) && (nodesCorrupt.contains(node))) ||
          node.isDecommissionInProgress() || node.isDecommissioned()) {
          // do nothing
        } else {
          live++;            // number of live nodes
          originalDatanodes.remove(node);
        }
      }

      if (originalDatanodes.size() > 0) {
        NameNode.stateChangeLog.info("Unable to delete excess replicas for block " +
          block +
          " because the state of the original replicas have changed." +
          " Will retry later.");
        overReplicatedBlocks.add(block);
        return;
      }
      
      short blockReplication = inode.getBlockReplication(storedBlock);

      // loop through datanodes that have excess-replicas of this block
      for (ListIterator<DatanodeID> iter = excessReplicateTmp.listIterator();
           iter.hasNext();) {
        DatanodeID datanodeId = iter.next();

        // re-check that block still has excess replicas.
        // If not, then there is nothing more to do.
        if (live <= blockReplication) {
          break;
        }

        // find the DatanodeDescriptor for this datanode
        DatanodeDescriptor datanode = null;
        try {
          datanode = getDatanode(datanodeId);
        } catch (IOException e) {
        }
        if (datanode == null) {
          NameNode.stateChangeLog.info("No datanode found while processing " +
            "overreplicated block " + block);
          continue;              // dead datanode?
        }

        // insert into excessReplicateMap
        LightWeightHashSet<Block> excessBlocks = excessReplicateMap.get(datanodeId.getStorageID());
        if (excessBlocks == null) {
          excessBlocks = new LightWeightHashSet<Block>();
          excessReplicateMap.put(datanodeId.getStorageID(), excessBlocks);
        }

        if (excessBlocks.add(block)) {
          excessBlocksCount++;
          if (NameNode.stateChangeLog.isDebugEnabled()) {
            NameNode.stateChangeLog.debug("BLOCK* NameSystem.chooseExcessReplicates: "
              + "(" + datanodeId.getName() + ", " + block
              + ") is added to excessReplicateMap");
          }
        }
        //
        // The 'excessblocks' tracks blocks until we get confirmation
        // that the datanode has deleted them; the only way we remove them
        // is when we get a "removeBlock" message.
        //
        // The 'invalidate' list is used to inform the datanode the block
        // should be deleted.  Items are removed from the invalidate list
        // upon giving instructions to the namenode.
        //
        addToInvalidatesNoLog(block, datanode, true);
        live--;
        if (NameNode.stateChangeLog.isDebugEnabled()) {
          NameNode.stateChangeLog.debug("BLOCK* NameSystem.chooseExcessReplicates: "
            + "(" + datanode.getName() + ", " + block + ") is added to recentInvalidateSets");
        }
      }
    } finally {
      writeUnlock();
    }
  }
337940
function dedupe(resources, onDupe) {
  // first create a map of name->[array of resources with same name]
  var names = {};
  resources.forEach(function(scriptResource) {
    if (names[scriptResource.name] == null) {
      names[scriptResource.name] = [];
    }
    names[scriptResource.name].push(scriptResource);
  });

  // Call the de-dupe function for each set of dupes
  Object.keys(names).forEach(function(name) {
    var clones = names[name];
    if (clones.length > 1) {
      onDupe(clones);
    }
  });
}
558864
protected void beginReplicating() {
        Log.v(TAG, "submit startReplicating()");
        executor.submit(new Runnable() {
            @Override
            public void run() {
                if (isRunning()) {
                    Log.v(TAG, "start startReplicating()");
                    initPendingSequences();
                    initDownloadsToInsert();
                    startChangeTracker();
                }
                // start replicator ...
            }
        });
    }
1230206
def header_deduplicator=(strategy)
      if header_deduplicator_registry.type?(strategy)
        @header_deduplicator = header_deduplicator_registry[strategy]
      elsif strategy.respond_to?(:call)
        @header_deduplicator = strategy
      else
        message = "unknown deduplication strategy: '#{strategy}'"
        raise(Errors::UnknownDeduplicationStrategyError, message)
      end
    end
148314
def PrependUnique(self, delete_existing=0, **kw):
        
        kw = copy_non_reserved_keywords(kw)
        for key, val in kw.items():
            if SCons.Util.is_List(val):
                val = _delete_duplicates(val, not delete_existing)
            if key not in self._dict or self._dict[key] in ('', None):
                self._dict[key] = val
            elif SCons.Util.is_Dict(self._dict[key]) and \
                 SCons.Util.is_Dict(val):
                self._dict[key].update(val)
            elif SCons.Util.is_List(val):
                dk = self._dict[key]
                if not SCons.Util.is_List(dk):
                    dk = [dk]
                if delete_existing:
                    dk = [x for x in dk if x not in val]
                else:
                    val = [x for x in val if x not in dk]
                self._dict[key] = val + dk
            else:
                dk = self._dict[key]
                if SCons.Util.is_List(dk):
                    # By elimination, val is not a list.  Since dk is a
                    # list, wrap val in a list first.
                    if delete_existing:
                        dk = [x for x in dk if x not in val]
                        self._dict[key] = [val] + dk
                    else:
                        if not val in dk:
                            self._dict[key] = [val] + dk
                else:
                    if delete_existing:
                        dk = [x for x in dk if x not in val]
                    self._dict[key] = val + dk
        self.scanner_map_delete(kw)
655685
private static void checkDuplicates(ImmutableList<SQLPPTriplesMap> mappings)
            throws DuplicateMappingException {

        Set<SQLPPTriplesMap> mappingSet = new HashSet<>(mappings);

        int duplicateCount = mappings.size() - mappingSet.size();

        /**
         * If there are some triplesMaps, finds them
         */
        if (duplicateCount > 0) {
            Set<String> duplicateIds = new HashSet<>();
            int remaining = duplicateCount;
            for (SQLPPTriplesMap mapping : mappings) {
                if (mappingSet.contains(mapping)) {
                    mappingSet.remove(mapping);
                }
                /**
                 * Duplicate
                 */
                else {
                    duplicateIds.add(mapping.getId());
                    if (--remaining == 0)
                        break;
                }
            }
            //TODO: indicate the source
            throw new DuplicateMappingException(String.format("Found %d duplicates in the following ids: %s",
                    duplicateCount, duplicateIds.toString()));
        }
    }