Index documents that are missing from ES index.

        Determines which documents are missing using ES `mget` call which
        returns a list of document IDs as `documents`. Then missing
        `documents` from that list are indexed.
174371
def index_missing_documents(self, documents, request=None):
        
        log.info('Trying to index documents of type `{}` missing from '
                 '`{}` index'.format(self.doc_type, self.index_name))
        if not documents:
            log.info('No documents to index')
            return
        query_kwargs = dict(
            index=self.index_name,
            doc_type=self.doc_type,
            fields=['_id'],
            body={'ids': [d['_pk'] for d in documents]},
        )
        try:
            response = self.api.mget(**query_kwargs)
        except IndexNotFoundException:
            indexed_ids = set()
        else:
            indexed_ids = set(
                d['_id'] for d in response['docs'] if d.get('found'))
        documents = [d for d in documents if str(d['_pk']) not in indexed_ids]

        if not documents:
            log.info('No documents of type `{}` are missing from '
                     'index `{}`'.format(self.doc_type, self.index_name))
            return

        self._bulk('index', documents, request)
891034
def _include_docs(self, rows):
        ''''''
        resp = list()
        for row in rows:
            if isinstance(row[1], dict) and '_id' in row[1]:
                d_id = row[1]['_id']
            else:
                d_id = row[2]
            try:
                doc = self._get_doc(d_id)
                doc = copy.deepcopy(doc)
                if doc.get('_deleted', None):
                    raise NotFoundError('%s deleted' % d_id)
            except NotFoundError:
                # FIXME: this just returns None instead of dict if the
                # document is not there/was deleted.
                # I'm not sure if this is how couchdb would behave
                # check it!
                doc = None
            resp.append(row + (doc, ))
        return resp
173453
def mget_list(item, index):
    ''
    if isinstance(index, (int, slice)):
        return item[index]
    else:
        return map(item.__getitem__, index)
369100
def search_update_index(self):
        
        doc_id = self.search_get_document_id(self.key)

        fields = [search.AtomField('class_name', name) for name in self.search_get_class_names()]

        index = self.search_get_index()

        if self.searchable_fields is None:
            searchable_fields = []

            for field, prop in self._properties.items():
                if field == 'class':
                    continue
                for class_, field_type in SEARCHABLE_PROPERTY_TYPES.items():
                    if isinstance(prop, class_):
                        searchable_fields.append(field)
        else:
            searchable_fields = self.searchable_fields

        for f in set(searchable_fields):
            prop = self._properties[f]
            value = getattr(self, f)
            field = None
            field_found = False
            for class_, field_type in SEARCHABLE_PROPERTY_TYPES.items():
                if isinstance(prop, class_):
                    field_found = True
                    if value is not None:
                        if isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set):
                            for v in value:
                                field = field_type(name=f, value=v)
                        elif isinstance(value, ndb.Key):
                            field = field_type(name=f, value=value.urlsafe())
                        else:
                            field = field_type(name=f, value=value)
            if not field_found:
                raise ValueError('Cannot find field type for %r on %r' % (prop, self.__class__))

            if field is not None:
                fields.append(field)

        document = search.Document(doc_id, fields=fields)
        index.put(document)
487996
def get_model_indices(cls, index):
        '''
        
        '''
        try:
            return cls._idx_name_to_mdl_to_mdlidx[index].values()
        except KeyError:
            raise KeyError('Could not find any index named {}. Is this index defined in BUNGIESEARCH["INDICES"]?'.format(index))
174613
def find_missing_modules():
    
    missing_modules = []

    for module in MODULES:
        try:
            __import__(module[1])
        except ImportError:
            missing_modules.append(module)
    return missing_modules
260603
def ids(cls, filter=None, **kwargs):
        """"""
        from mongoframes.queries import Condition, Group, to_refs

        # Find the documents
        if isinstance(filter, (Condition, Group)):
            filter = filter.to_dict()

        documents = cls.get_collection().find(
            to_refs(filter),
            projection={'_id': True},
            **kwargs
            )

        return [d['_id'] for d in list(documents)]
1291479
private void esIndex(String index, String id, String json, String pipeline) {
        logger.debug("Indexing {}/{}?pipeline={}", index, id, pipeline);
        logger.trace("JSon indexed : {}", json);

        if (!closed) {
            esClient.index(index, id, json, pipeline);
        } else {
            logger.warn("trying to add new file while closing crawler. Document [{}]/[{}] has been ignored", index, id);
        }
    }
1349301
def get_document_list(self):
        
        try:
            return self.__dict__['document_list']
        except KeyError:
            obj_list = DocumentSet([
                self._connection.documents.get(i) for i in self.document_ids
            ])
            self.__dict__['document_list'] = obj_list
            return obj_list
516792
def index_objects(self, objects, index="default"):
        """"""
        if not objects:
            return

        index_name = index
        index = self.app_state.indexes[index_name]
        indexed = set()

        with index.writer() as writer:
            for obj in objects:
                document = self.get_document(obj)
                if document is None:
                    continue

                object_key = document["object_key"]
                if object_key in indexed:
                    continue

                writer.delete_by_term("object_key", object_key)
                try:
                    writer.add_document(**document)
                except ValueError:
                    # logger is here to give us more infos in order to catch a weird bug
                    # that happens regularly on CI but is not reliably
                    # reproductible.
                    logger.error("writer.add_document(%r)", document, exc_info=True)
                    raise
                indexed.add(object_key)
874711
def detect_missing_relations(self, obj, exc):
        
        missing = defaultdict(set)
        for name, err in exc.error_dict.items():
            # check if it was a relationship that doesnt exist locally
            pattern = r".+ with id (\d+) does not exist.+"
            m = re.match(pattern, str(err))
            if m:
                field = obj._meta.get_field(name)
                res = self.get_resource(field.related_model)
                missing[res].add(int(m.group(1)))
        return missing
721081
def ensure_index(self, index, mappings=None, settings=None, clear=False):
        
        mappings = mappings or []
        if isinstance(mappings, dict):
            mappings = [mappings]
        exists = self.indices.exists_index(index)
        if exists and not mappings and not clear:
            return
        if exists and clear:
            self.indices.delete_index(index)
            exists = False

        if exists:
            if not mappings:
                self.indices.delete_index(index)
                self.indices.refresh()
                self.indices.create_index(index, settings)
                return

            if clear:
                for maps in mappings:
                    for key in list(maps.keys()):
                        self.indices.delete_mapping(index, doc_type=key)
                self.indices.refresh()
            if isinstance(mappings, SettingsBuilder):
                for name, data in list(mappings.mappings.items()):
                    self.indices.put_mapping(doc_type=name, mapping=data, indices=index)

            else:
                from pyes.mappings import DocumentObjectField, ObjectField

                for maps in mappings:
                    if isinstance(maps, tuple):
                        name, mapping = maps
                        self.indices.put_mapping(doc_type=name, mapping=mapping, indices=index)
                    elif isinstance(maps, dict):
                        for name, data in list(maps.items()):
                            self.indices.put_mapping(doc_type=name, mapping=maps, indices=index)
                    elif isinstance(maps, (DocumentObjectField, ObjectField)):
                        self.put_mapping(doc_type=maps.name, mapping=maps.as_dict(), indices=index)

                return

        if settings:
            if isinstance(settings, dict):
                settings = SettingsBuilder(settings, mappings)
        else:
            if isinstance(mappings, SettingsBuilder):
                settings = mappings
            else:
                settings = SettingsBuilder(mappings=mappings)
        if not exists:
            self.indices.create_index(index, settings)
            self.indices.refresh(index, timesleep=1)
960494
def get_documents(self, term):
        
        if term not in self._terms:
            raise IndexError(TERM_DOES_NOT_EXIST)
        else:
            return self._terms[term]
1202403
private void indexMissingFields() {
      for (FieldDescriptor fieldDescriptor : messageContext.getMessageDescriptor().getFields()) {
         if (!messageContext.isFieldMarked(fieldDescriptor.getNumber())) {
            Object defaultValue = fieldDescriptor.getJavaType() == JavaType.MESSAGE
                  || fieldDescriptor.hasDefaultValue() ? fieldDescriptor.getDefaultValue() : null;
            IndexingMetadata indexingMetadata = messageContext.getMessageDescriptor().getProcessedAnnotation(IndexingMetadata.INDEXED_ANNOTATION);
            FieldMapping fieldMapping = indexingMetadata != null ? indexingMetadata.getFieldMapping(fieldDescriptor.getName()) : null;
            if (indexingMetadata == null && isLegacyIndexingEnabled || fieldMapping != null && fieldMapping.index()) {
               addFieldToDocument(fieldDescriptor, defaultValue, fieldMapping);
            }
         }
      }
   }
144247
def set_doc_ids(self, doc_ids):
        
        if isinstance(doc_ids, list):
            self.set_documents(dict.fromkeys(doc_ids))
        else:
            self.set_documents({doc_ids: None})
1129957
def get_document(self, id):
        
        obj_list = self.document_list
        matches = [i for i in obj_list if str(i.id) == str(id)]
        if not matches:
            raise DoesNotExistError("The resource you've requested does not \
exist or is unavailable without the proper credentials.")
        return matches[0]
662827
function find(doc, element, id) {
  if (!element) {
    element = doc.getElementById(id);
    if (!element) {
      throw new Error('Missing element, id=' + id);
    }
  }
  return element;
}