Remove any duplicate records with identical IDs, keep the first
    instance seen and discard additional occurences.
355357
def deduplicate_taxa(records):
    
    logging.info('Applying _deduplicate_taxa generator: ' + \
                 'removing any duplicate records with identical IDs.')
    taxa = set()
    for record in records:
        # Default to full ID, split if | is found.
        taxid = record.id
        if '|' in record.id:
            try:
                taxid = int(record.id.split("|")[0])
            except:
                # If we couldn't parse an integer from the ID, just fall back
                # on the ID
                logging.warn("Unable to parse integer taxid from %s",
                        taxid)
        if taxid in taxa:
            continue
        taxa.add(taxid)
        yield record
922486
def _remove_duplicates(self):
        
        contained = set()
        for m1, m2 in itertools.combinations(self._max_rects, 2):
            if m1.contains(m2):
                contained.add(m2)
            elif m2.contains(m1):
                contained.add(m1)
        
        # Remove from max_rects
        self._max_rects = [m for m in self._max_rects if m not in contained]
520356
def remove_duplicates(slist):
    """"""
    unique = list(set(slist))
    difference = len(slist) - len(unique)
    if difference == 1:
        write_message("Removed one duplicate entry from input list.\n")
    if difference > 1:
        write_message("Removed %i duplicate entries from input list.\n" % difference)
    return unique
409197
public function removeProductDuplicates() {
    $this->output->writeln(dt('Removing duplicates in commerce products, please wait...'));

    $skus_to_sync = [];

    /** @var \Drupal\node\NodeStorageInterface $nodeStorage */
    $nodeStorage = $this->entityTypeManager->getStorage('node');

    $query = $this->connection->select('acm_sku_field_data', 't1');
    $query->addField('t1', 'id', 'id');
    $query->addField('t1', 'sku', 'sku');
    $query->leftJoin('acm_sku_field_data', 't2', 't1.sku = t2.sku');
    $query->where('t1.id != t2.id');
    $result = $query->execute()->fetchAllKeyed(0, 1);

    if (empty($result)) {
      $this->output->writeln(dt('No duplicate skus found.'));
    }
    else {
      $skus = [];

      foreach ($result as $id => $sku) {
        $skus[$sku][$id] = $id;
        $skus_to_sync[$sku] = $sku;
      }

      foreach ($skus as $sku => $ids) {
        $this->output->writeln(dt('Duplicate skus found for sku: @sku with ids: @ids.', [
          '@sku' => $sku,
          '@ids' => implode(', ', $ids),
        ]));

        // Always delete the one with higher id, first one will have more
        // translations.
        sort($ids);

        // Remove the first id which we don't want to delete.
        array_shift($ids);

        foreach ($ids as $id) {
          $this->output->writeln(dt('Deleting sku with id @id for sku @sku.', [
            '@sku' => $sku,
            '@id' => $id,
          ]));

          $sku_entity = SKU::load($id);
          $sku_entity->delete();
        }
      }
    }

    $query = $this->connection->select('node__' . $this->skuFieldName, 't1');
    $query->addField('t1', 'entity_id', 'id');
    $query->addField('t1', $this->skuFieldName . '_value', 'sku');
    $query->leftJoin('node__' . $this->skuFieldName, 't2', 't1.' . $this->skuFieldName . '_value = t2.' . $this->skuFieldName . '_value');
    $query->where('t1.entity_id != t2.entity_id');
    $result = $query->execute()->fetchAllKeyed(0, 1);

    if (empty($result)) {
      $this->output->writeln(dt('No duplicate product nodes found.'));
    }
    else {
      $nids_to_delete = [];
      $skus = [];

      foreach ($result as $id => $sku) {
        $skus[$sku][$id] = $id;
        $skus_to_sync[$sku] = $sku;
      }

      foreach ($skus as $sku => $ids) {
        $this->output->writeln(dt('Duplicate nodes found for sku: @sku with ids: @ids.', [
          '@sku' => $sku,
          '@ids' => implode(', ', $ids),
        ]));

        // Always delete the one with higher nid, first one will have proper
        // url alias.
        sort($ids);

        // Remove the first id which we don't want to delete.
        array_shift($ids);

        foreach ($ids as $id) {
          $this->output->writeln(dt('Deleting node with id @id for sku @sku.', [
            '@sku' => $sku,
            '@id' => $id,
          ]));

          $nids_to_delete[$id] = $id;
        }
      }

      if ($nids_to_delete) {
        $nodeStorage->delete($nodeStorage->loadMultiple($nids_to_delete));
      }
    }

    if ($skus_to_sync) {
      $sku_texts = implode(',', $skus_to_sync);

      $this->output->writeln(dt('Requesting resync for skus @skus.', [
        '@skus' => $sku_texts,
      ]));

      foreach ($this->i18nhelper->getStoreLanguageMapping() as $langcode => $store_id) {
        // Using very small page size to avoid any issues for skus which already
        // had corrupt data.
        $this->ingestApiWrapper->productFullSync($store_id, $langcode, $sku_texts, NULL, 5);
      }
    }

    $this->output->writeln(dt('Done.'));
  }
385374
def remove_duplicate_edges_directed(dg):
    """"""
    # With directed edges, we can just hash the to and from node id tuples and if
    # a node happens to conflict with one that already exists, we delete it

    # --For aesthetic, we sort the edge ids so that lower edge ids are kept
    lookup = {}
    edges = sorted(dg.get_all_edge_ids())
    for edge_id in edges:
        e = dg.get_edge(edge_id)
        tpl = e['vertices']
        if tpl in lookup:
            dg.delete_edge_by_id(edge_id)
        else:
            lookup[tpl] = edge_id
419593
def diff_record(self, record):
        """"""
        rec = self.get_record(record._record_type, record.host)
        if rec is not None and record is not None:
            return {'removed': tuple(set(rec.results) - set(record.results)),
                    'added': tuple(set(record.results) - set(rec.recults))}
        else:
            return False
706637
def remove_duplicate_edges_undirected(udg):
    """"""
    # With undirected edges, we need to hash both combinations of the to-from node ids, since a-b and b-a are equivalent
    # --For aesthetic, we sort the edge ids so that lower edges ids are kept
    lookup = {}
    edges = sorted(udg.get_all_edge_ids())
    for edge_id in edges:
        e = udg.get_edge(edge_id)
        tpl_a = e['vertices']
        tpl_b = (tpl_a[1], tpl_a[0])
        if tpl_a in lookup or tpl_b in lookup:
            udg.delete_edge_by_id(edge_id)
        else:
            lookup[tpl_a] = edge_id
            lookup[tpl_b] = edge_id
1169728
def delete(self,removed):
        
        if isinstance(removed,dict):
            # remove a single record
            removed = [removed]
        else:
            # convert iterable into a list (to be able to sort it)
            removed = [ r for r in removed ]
        if not removed:
            return 0
        _ids = [ r['__id__'] for r in removed ]
        _ids.sort()
        sql = "DELETE FROM %s WHERE __id__ IN (%s)" %(self.name,
            ",".join([str(_id) for _id in _ids]))
        self.cursor.execute(sql)
        return len(removed)
136326
def repair_duplicate_names(self):
		
		for name in self._all_names():
			cur = self.db.find({'names': name})
			main_doc = next(cur)
			for duplicate in cur:
				query = {'_id': main_doc['_id']}
				update = {
					'$inc': {'value': duplicate['value']},
					'$push': {'names': {'$each': duplicate['names']}},
				}
				self.db.update(query, update)
				self.db.remove(duplicate)
1010903
public function unique()
    {
        $exist = $remove = array();

        $old = $this->_filter;
        $this->setIteratorFilter(self::HIDE_GROUPS | self::BASE_ELEMENTS);

        // For duplicates, we use the first address that contains personal
        // information.
        foreach ($this as $key => $val) {
            $bare = $val->bare_address;
            if (isset($exist[$bare])) {
                if (($exist[$bare] == -1) || is_null($val->personal)) {
                    $remove[] = $key;
                } else {
                    $remove[] = $exist[$bare];
                    $exist[$bare] = -1;
                }
            } else {
                $exist[$bare] = is_null($val->personal)
                    ? $key
                    : -1;
            }
        }

        foreach (array_reverse($remove) as $key) {
            unset($this[$key]);
        }

        $this->_filter = $old;
    }
803508
def pop(self, instance):
        '''
'''
        if isinstance(instance, self.model):
            iid = instance.get_state().iid
        else:
            iid = instance
        instance = None
        for d in (self._new, self._modified, self._deleted):
            if iid in d:
                inst = d.pop(iid)
                if instance is None:
                    instance = inst
                elif inst is not instance:
                    raise ValueError('Critical error: %s is duplicated' % iid)
        return instance
505695
def remove_duplicate_notes(self):
        """"""
        res = []
        for x in self.notes:
            if x not in res:
                res.append(x)
        self.notes = res
        return res
1172034
function checkForDuplicateIds( page ) {
      const idCount = {};
      forEachArea( page, items => {
         items.forEach( ({ id }) => {
            idCount[ id ] = idCount[ id ] ? idCount[ id ] + 1 : 1;
         } );
      } );

      const duplicates = Object.keys( idCount ).filter( id => idCount[ id ] > 1 );
      if( duplicates.length ) {
         throw formatError( page, `Duplicate widget/composition/layout ID(s): ${duplicates.join( ', ' )}` );
      }
   }
1022690
def remove_duplicates(seq, idfun=None):
    
    # order preserving
    if idfun is None:

        def idfun(x):
            return x

    seen = {}
    result = []
    for item in seq:
        marker = idfun(item)
        # in old Python versions:
        # if seen.has_key(marker)
        # but in new ones:
        if marker in seen: continue
        seen[marker] = 1
        result.append(item)
    return result
342276
public function removePendingJobDuplicates()
    {
        $pdo = $this->getEntityManager()->getPDO();

        $query = "
            SELECT scheduled_job_id
            FROM job
            WHERE
                scheduled_job_id IS NOT NULL AND
                `status` = '".CronManager::PENDING."' AND
                execute_time <= '".date('Y-m-d H:i:s')."' AND
                target_id IS NULL AND
                deleted = 0
            GROUP BY scheduled_job_id
            HAVING count( * ) > 1
            ORDER BY MAX(execute_time) ASC
        ";
        $sth = $pdo->prepare($query);
        $sth->execute();

        $duplicateJobList = $sth->fetchAll(PDO::FETCH_ASSOC);

        foreach ($duplicateJobList as $row) {
            if (!empty($row['scheduled_job_id'])) {

                $query = "
                    SELECT id FROM `job`
                    WHERE
                        scheduled_job_id = ".$pdo->quote($row['scheduled_job_id'])."
                        AND `status` = '" . CronManager::PENDING ."'
                        ORDER BY execute_time
                        DESC LIMIT 1, 100000
                    ";
                $sth = $pdo->prepare($query);
                $sth->execute();
                $jobIdList = $sth->fetchAll(PDO::FETCH_COLUMN);

                if (empty($jobIdList)) {
                    continue;
                }

                $quotedJobIdList = [];
                foreach ($jobIdList as $jobId) {
                    $quotedJobIdList[] = $pdo->quote($jobId);
                }

                $update = "
                    UPDATE job
                    SET deleted = 1
                    WHERE
                        id IN (".implode(", ", $quotedJobIdList).")
                ";

                $sth = $pdo->prepare($update);
                $sth->execute();
            }
        }
    }
1051367
public function duplicate($new_id = null)
    {
        $this->id = null;

        if ($this->id_field) {
            $this[$this->id_field] = $new_id;
        }

        return $this;
    }
849012
def _duplicate_queries(self, output):
        """"""
        if QC_SETTINGS['DISPLAY_DUPLICATES']:
            for query, count in self.queries.most_common(QC_SETTINGS['DISPLAY_DUPLICATES']):
                lines = ['\nRepeated {0} times.'.format(count)]
                lines += wrap(query)
                lines = "\n".join(lines) + "\n"
                output += self._colorize(lines, count)
        return output