filters the import stream to remove duplicates

        also serves as a good place to override if anything special has to be done to the
        order of the import stream (see OrganizationImporter)
160800
def _prepare_imports(self, dicts):

        
        # hash(json): id
        seen_hashes = {}

        for data in dicts:
            json_id = data.pop('_id')

            # map duplicates (using omnihash to tell if json dicts are identical-ish)
            objhash = omnihash(data)
            if objhash not in seen_hashes:
                seen_hashes[objhash] = json_id
                yield json_id, data
            else:
                self.duplicates[json_id] = seen_hashes[objhash]
258213
private boolean cleanupInventory() {
		boolean modified = false;
		modified |= removeUnreadable();
		modified |= recreateMismatching();
		modified |= importConfigurations();
		return modified;
	}
411711
function getOveruse(duplicates, limit) {
    var duplicate;
    for (duplicate in duplicates) {
        if (duplicates[duplicate] < limit) {
            delete duplicates[duplicate]
        }
    }
    return keys(duplicates);
}
632300
def remove_excluded(self):
        
        # import yaml
        # print yaml.dump({k:v.__json__() for k,v in self.sources.items()}, default_flow_style=False)
        sources = list(self.sources.values())
        for src in sources:
            if src.excluded:
                del self.sources[src.name]
            src.imports = [m for m in src.imports if not self._exclude(m)]
            src.imported_by = [m for m in src.imported_by if not self._exclude(m)]
378662
function (newFilters, oldFilters) {
        var self = this;

        if (!oldFilters) {
            oldFilters = this._filters;
        } else if (!isArray(oldFilters)) {
            oldFilters = [oldFilters];
        }

        if (!newFilters) {
            newFilters = [];
        } else if (!isArray(newFilters)) {
            newFilters = [newFilters];
        }

        oldFilters.forEach(function (filter) {
            self._removeFilter(filter);
        });

        newFilters.forEach(function (filter) {
            self._addFilter(filter);
        });

        this._runFilters();
    }
595384
function(obj, key, channel, sysImports, next) {
    if (obj) {
      var self = this,
        modelName = this.getDataSourceName('dup'),
        objVal = helper.JSONPath(obj, key),
        filter = {
          owner_id : channel.owner_id,
          channel_id : channel.id,
          bip_id : sysImports.bip && sysImports.bip.id,
          value : objVal
        },
        props = {
          last_update : helper.nowUTCMS(),
          owner_id : channel.owner_id,
          channel_id : channel.id,
          bip_id : sysImports.bip && sysImports.bip.id,
          value : objVal
        };

      self.dao.find(modelName, filter, function(err, result) {
        if (err) {
          next(err);
        } else {
          if (!result || (result && result.value != objVal)) {
            self.dao.upsert(modelName, filter, props, function(err, result) {
              next(err, obj);
            });
          }
        }
      });
    }
  }
990884
def purge_duplicates(list_in):
    
    _list = []
    for item in list_in:
        if item not in _list:
            _list.append(item)
    return _list
1130031
@Override
    public ZipImporter importFrom(final File file) throws ArchiveImportException {
        return importFrom(file, Filters.includeAll());
    }
779680
public List<Import> getPublicImports() {
        return getImports()
                .stream()
                .filter(Import::isPublic)
                .collect(Collectors.toList());
    }
341135
@Nonnull
	public Imports filter() {
		return new Imports(Sets.newHashSet(Collections2.filter(imports, Predicates.and(IGNORE_JAVA_LANG, IGNORE_UNDEFINED))));
	}
242625
@Override
    public ZipImporter importFrom(InputStream stream, Filter<ArchivePath> filter) throws ArchiveImportException {
        Validate.notNull(stream, "Stream must be specified");
        Validate.notNull(filter, "Filter must be specified");

        try {
            // Wrap in ZipInputStream if we haven't been given one
            final ZipInputStream zipStream = new ZipInputStream(stream);

            ZipEntry entry;
            while ((entry = zipStream.getNextEntry()) != null) {
                // Get the name
                final String entryName = entry.getName();

                if(!filter.include(ArchivePaths.create(entryName))) {
                    zipStream.closeEntry();
                    continue;
                }

                // Get the archive
                final Archive<?> archive = this.getArchive();

                // Handle directories separately
                if (entry.isDirectory()) {
                    archive.addAsDirectory(entryName);
                    continue;
                }

                final ByteArrayOutputStream output = new ByteArrayOutputStream(8192);
                IOUtil.copy(zipStream, output);
                archive.add(new ByteArrayAsset(output.toByteArray()), entryName);
                zipStream.closeEntry();
            }
        } catch (IOException e) {
            throw new ArchiveImportException("Could not import stream", e);
        }
        return this;
    }
887785
function(options){
            var cleanOptions = this.parseOptions(options),
                excludes = _.clone(this.privateFields),
                includeExludes = [];

            // If any includes are passed in through the options clean up the
            // default excluded fields. A type can have some excluded fields by
            // default and if the user overrides this then we need to clean up the query.
            if(options && options.include){
                _.each(options.include, function(include){
                    //If the include matches an existing exclude then remove it from the
                    //excludes and also from the includes (you can't mix them)
                    excludes = _.remove(excludes, function(o){
                        if(o === include){
                            cleanOptions.include = _.remove(cleanOptions.include, function(i){
                                return i !== include;
                            });
                        }

                        return o !== include;
                    });

                });
            }

            includeExludes = _.union(cleanOptions.include,
                this.parseExclusion(cleanOptions.exclude),
                this.parseExclusion(excludes)
            );

            return includeExludes.join(' ');
        }
1188964
function clearImports(partials) {
        for (var partialName in partials) {
            var _imports = partials[partialName].imports;

            for (var index in _imports) {
                var _partialToRemove = _imports[index];
                var i = 0;
                // remove file from list
                delete mainPartialList[_partialToRemove];
            } // end in imports
        }// end in files
    }
1054121
public void addAllLocalisationsForCleanUp(boolean singleServer,HashMap postMediatedItemStreamsRequiringCleanup)
  {
    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled())
      SibTr.entry(tc,
                  "addAllLocalisationsForCleanUp" ,
                  new Object[]{
                  Boolean.valueOf(singleServer)});

    // If _xmitQueuePoints not null, then this is a Queue
    if (_xmitQueuePoints != null)
    {
      synchronized(_xmitQueuePoints)
      {
        {
          Iterator itr = _xmitQueuePoints.values().iterator();
          while(itr.hasNext())
          {
            PtoPXmitMsgsItemStream xmitQueue = (PtoPXmitMsgsItemStream)itr.next();
            postMediatedItemStreamsRequiringCleanup.put(xmitQueue.getLocalizingMEUuid(), xmitQueue);
          }
        }
      }
    }

    

    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled())
      SibTr.exit(tc, "addAllLocalisationsForCleanUp");
  }
1022690
def remove_duplicates(seq, idfun=None):
    
    # order preserving
    if idfun is None:

        def idfun(x):
            return x

    seen = {}
    result = []
    for item in seq:
        marker = idfun(item)
        # in old Python versions:
        # if seen.has_key(marker)
        # but in new ones:
        if marker in seen: continue
        seen[marker] = 1
        result.append(item)
    return result
31237
def _remove_finder(importer, finder):
  """"""

  existing_finder = _get_finder(importer)

  if not existing_finder:
    return

  if isinstance(existing_finder, ChainedFinder):
    try:
      existing_finder.finders.remove(finder)
    except ValueError:
      return
    if len(existing_finder.finders) == 1:
      pkg_resources.register_finder(importer, existing_finder.finders[0])
    elif len(existing_finder.finders) == 0:
      pkg_resources.register_finder(importer, pkg_resources.find_nothing)
  else:
    pkg_resources.register_finder(importer, pkg_resources.find_nothing)
784931
function (file, importOptions = {}) {
        var self = this;

        // Step 1: Handle converting the file to usable data
        return this.loadFile(file).then(function (importData) {
            // Step 2: Let the importers pre-process the data
            return self.preProcess(importData);
        }).then(function (importData) {
            // Step 3: Actually do the import
            // @TODO: It would be cool to have some sort of dry run flag here
            return self.doImport(importData, importOptions);
        }).then(function (importData) {
            // Step 4: Report on the import
            return self.generateReport(importData);
        }).finally(() => self.cleanUp()); // Step 5: Cleanup any files
    }