Permanently index all documents previously added via `buffer`, or
        directly index documents from `corpus`, if specified.

        The indexing model must already exist (see `train`) before this function
        is called.
502329
def index(self, corpus=None, clear_buffer=True):
        
        if not self.model:
            msg = 'must initialize model for %s before indexing documents' % self.basename
            logger.error(msg)
            raise AttributeError(msg)

        if corpus is not None:
            # use the supplied corpus only (erase existing buffer, if any)
            self.flush(clear_buffer=True)
            self.buffer(corpus)

        if not self.fresh_docs:
            msg = "index called but no indexing corpus specified for %s" % self
            logger.error(msg)
            raise ValueError(msg)

        if not self.fresh_index:
            logger.info("starting a new fresh index for %s" % self)
            self.fresh_index = SimIndex(self.location('index_fresh'), self.model.num_features)
        self.fresh_index.index_documents(self.fresh_docs, self.model)
        if self.opt_index is not None:
            self.opt_index.delete(self.fresh_docs.keys())
        logger.info("storing document payloads")
        for docid in self.fresh_docs:
            payload = self.fresh_docs[docid].get('payload', None)
            if payload is None:
                # HACK: exit on first doc without a payload (=assume all docs have payload, or none does)
                break
            self.payload[docid] = payload
        self.flush(save_index=True, clear_buffer=clear_buffer)
459841
def index_model(index_name, adapter):
    ''' '''
    model = adapter.model
    log.info('Indexing {0} objects'.format(model.__name__))
    qs = model.objects
    if hasattr(model.objects, 'visible'):
        qs = qs.visible()
    if adapter.exclude_fields:
        qs = qs.exclude(*adapter.exclude_fields)

    docs = iter_qs(qs, adapter)
    docs = iter_for_index(docs, index_name)

    for ok, info in streaming_bulk(es.client, docs, raise_on_error=False):
        if not ok:
            log.error('Unable to index %s "%s": %s', model.__name__,
                      info['index']['_id'], info['index']['error'])
472618
function (doc) {
    var self = this;
    var id = doc._id;
    if (self._published.has(id))
      throw Error("tried to add something already published " + id);
    if (self._limit && self._unpublishedBuffer.has(id))
      throw Error("tried to add something already existed in buffer " + id);

    var limit = self._limit;
    var comparator = self._comparator;
    var maxPublished = (limit && self._published.size() > 0) ?
      self._published.get(self._published.maxElementId()) : null;
    var maxBuffered = (limit && self._unpublishedBuffer.size() > 0) ?
      self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId()) : null;
    // The query is unlimited or didn't publish enough documents yet or the new
    // document would fit into published set pushing the maximum element out,
    // then we need to publish the doc.
    var toPublish = ! limit || self._published.size() < limit ||
                    comparator(doc, maxPublished) < 0;

    // Otherwise we might need to buffer it (only in case of limited query).
    // Buffering is allowed if the buffer is not filled up yet and all matching
    // docs are either in the published set or in the buffer.
    var canAppendToBuffer = !toPublish && self._safeAppendToBuffer &&
                            self._unpublishedBuffer.size() < limit;

    // Or if it is small enough to be safely inserted to the middle or the
    // beginning of the buffer.
    var canInsertIntoBuffer = !toPublish && maxBuffered &&
                              comparator(doc, maxBuffered) <= 0;

    var toBuffer = canAppendToBuffer || canInsertIntoBuffer;

    if (toPublish) {
      self._addPublished(id, doc);
    } else if (toBuffer) {
      self._addBuffered(id, doc);
    } else {
      // dropping it and not saving to the cache
      self._safeAppendToBuffer = false;
    }
  }
967608
private void saveFeatureIndexes(FeatureIndex[] index, String outputDir) throws IOException
	{
		logger.info("");

		// save the indexes
		for (int i=0;i<index.length;i++)
		{
			logger.debug("dic" + i + " size " + index[i].size());

			File tmp = new File(outputDir + "dic" + i);
			//File tmp = File.createTempFile("dic" + i, null);
			//tmp.deleteOnExit();

			BufferedWriter bwd = new BufferedWriter(new FileWriter(tmp));
			index[i].write(bwd);
			bwd.close();

			// add the dictionary to the model
			//model.add(tmp, "dic" + i);
		} // end for
	}
1103998
def unindex_model_on_delete(sender, document, **kwargs):
    ''''''
    if current_app.config.get('AUTO_INDEX'):
        unindex.delay(document)
492979
void refresh()
    {
        ListIterator<Document<K>> listIterator = documents_.listIterator();
        while (listIterator.hasNext())
        {
            if (listIterator.next() == null)
                listIterator.remove();
        }
    }
575773
private void createIndex(long idleTime) {
        try {
            this.sessions.createIndex(
                    new Document(SESSION_TTL, 1),
                    new IndexOptions()
                    .expireAfter(idleTime, TimeUnit.SECONDS)
                    .name(SESSION_INDEX_NAME));
        } catch (MongoException ex) {//update idle time
            this.sessions.dropIndex(SESSION_INDEX_NAME);
            this.sessions.createIndex(
                    new Document(SESSION_TTL, 1),
                    new IndexOptions()
                    .expireAfter(idleTime, TimeUnit.SECONDS)
                    .name(SESSION_INDEX_NAME));
        }
    }
284699
def _reindex_with_indexers(self, reindexers, fill_value=None, copy=False,
                               allow_dups=False):
        

        # reindex doing multiple operations on different axes if indicated
        new_data = self._data
        for axis in sorted(reindexers.keys()):
            index, indexer = reindexers[axis]
            baxis = self._get_block_manager_axis(axis)

            if index is None:
                continue

            index = ensure_index(index)
            if indexer is not None:
                indexer = ensure_int64(indexer)

            # TODO: speed up on homogeneous DataFrame objects
            new_data = new_data.reindex_indexer(index, indexer, axis=baxis,
                                                fill_value=fill_value,
                                                allow_dups=allow_dups,
                                                copy=copy)

        if copy and new_data is self._data:
            new_data = new_data.copy()

        return self._constructor(new_data).__finalize__(self)
240852
def create_indexed_document(index_instance, model_items, action):
    '''
    
    '''
    data = []
    if action == 'delete':
        for pk in model_items:
            data.append({'_id': pk, '_op_type': action})
    else:
        for doc in model_items:
            if index_instance.matches_indexing_condition(doc):
                data.append(index_instance.serialize_object(doc))
    return data
526706
def delete_index(self, refresh=False, ignore=None):
        """"""
        es = connections.get_connection("default")
        index = self.__class__.search_objects.mapping.index
        doc_type = self.__class__.search_objects.mapping.doc_type
        es.delete(index, doc_type, id=self.pk, refresh=refresh, ignore=ignore)
261674
def index_run(record_path, keep_json, check_duplicate):
    
    from .config import ConfigStore
    from .indexer import Indexer
    cfstore = ConfigStore()
    indexer = Indexer(cfstore, check_duplicate, keep_json, record_path)
    indexer.index_all()
552653
private void indexQuery(
      long index,
      long timestamp,
      RaftSession session,
      PrimitiveOperation operation,
      CompletableFuture<OperationResult> future) {
    // If the query index is greater than the session's last applied index, queue the request for handling once the
    // state machine is caught up.
    if (index > currentIndex) {
      log.trace("Registering query with index " + index + " > " + currentIndex);
      session.registerIndexQuery(index, () -> applyQuery(timestamp, session, operation, future));
    } else {
      applyQuery(timestamp, session, operation, future);
    }
  }
607156
public function endBulkIndex() {
        if (!$this->connected) {
            return;
        }

		$index = $this->getIndex();

        try {
            foreach ($this->buffer as $type => $documents) {
                $index->getType($type)->addDocuments($documents);
                $index->refresh();
            }
        } catch (HttpException $ex) {
            $this->connected = false;

            // TODO LOG THIS ERROR
            \SS_Log::log($ex, \SS_Log::ERR);
        } catch (\Elastica\Exception\BulkException $be) {
            \SS_Log::log($be, \SS_Log::ERR);
            throw $be;
        }

		$this->buffered = false;
		$this->buffer = array();
	}
251554
def index(self, text, terms=None, **kwargs):

        

        self.clear()

        # By default, use all terms.
        terms = terms or text.terms.keys()

        pairs = combinations(terms, 2)
        count = comb(len(terms), 2)

        for t1, t2 in bar(pairs, expected_size=count, every=1000):

            # Set the Bray-Curtis distance.
            score = text.score_braycurtis(t1, t2, **kwargs)
            self.set_pair(t1, t2, score)
535150
def index_all(self):
        
        self.logger.debug('Start indexing all records under: %s',
                          self.record_path)
        with self.db.connection():
            for json_path in sorted(self.find_record_files()):
                self.index_record(json_path)
112968
def _index_document(self, identifier, force=False):
        
        writer = self.index.writer()
        all_names = set([x['name'] for x in self.index.searcher().documents()])
        if identifier['name'] not in all_names:
            writer.add_document(**identifier)
            writer.commit()
516792
def index_objects(self, objects, index="default"):
        """"""
        if not objects:
            return

        index_name = index
        index = self.app_state.indexes[index_name]
        indexed = set()

        with index.writer() as writer:
            for obj in objects:
                document = self.get_document(obj)
                if document is None:
                    continue

                object_key = document["object_key"]
                if object_key in indexed:
                    continue

                writer.delete_by_term("object_key", object_key)
                try:
                    writer.add_document(**document)
                except ValueError:
                    # logger is here to give us more infos in order to catch a weird bug
                    # that happens regularly on CI but is not reliably
                    # reproductible.
                    logger.error("writer.add_document(%r)", document, exc_info=True)
                    raise
                indexed.add(object_key)