Insert multiple documents into Elasticsearch.
342326
def bulk_upsert(self, docs, namespace, timestamp):
        """"""
        def docs_to_upsert():
            doc = None
            for doc in docs:
                # Remove metadata and redundant _id
                index, doc_type = self._index_and_mapping(namespace)
                doc_id = u(doc.pop("_id"))
                document_action = {
                    '_index': index,
                    '_type': doc_type,
                    '_id': doc_id,
                    '_source': self._formatter.format_document(doc)
                }
                document_meta = {
                    '_index': self.meta_index_name,
                    '_type': self.meta_type,
                    '_id': doc_id,
                    '_source': {
                        'ns': namespace,
                        '_ts': timestamp
                    }
                }
                yield document_action
                yield document_meta
            if doc is None:
                raise errors.EmptyDocsError(
                    "Cannot upsert an empty sequence of "
                    "documents into Elastic Search")
        try:
            kw = {}
            if self.chunk_size > 0:
                kw['chunk_size'] = self.chunk_size

            responses = streaming_bulk(client=self.elastic,
                                       actions=docs_to_upsert(),
                                       **kw)

            for ok, resp in responses:
                if not ok:
                    LOG.error(
                        "Could not bulk-upsert document "
                        "into ElasticSearch: %r" % resp)
            if self.auto_commit_interval == 0:
                self.commit()
        except errors.EmptyDocsError:
            # This can happen when mongo-connector starts up, there is no
            # config file, but nothing to dump
            pass
677389
def upsert(self, doc, namespace, timestamp, update_spec=None):
        """"""
        index, doc_type = self._index_and_mapping(namespace)
        # No need to duplicate '_id' in source document
        doc_id = u(doc.pop("_id"))
        metadata = {
            'ns': namespace,
            '_ts': timestamp
        }

        # Index the source document, using lowercase namespace as index name.
        action = {
            '_op_type': 'index',
            '_index': index,
            '_type': doc_type,
            '_id': doc_id,
            '_source': self._formatter.format_document(doc)
        }
        # Index document metadata with original namespace (mixed upper/lower).
        meta_action = {
            '_op_type': 'index',
            '_index': self.meta_index_name,
            '_type': self.meta_type,
            '_id': doc_id,
            '_source': bson.json_util.dumps(metadata)
        }

        self.index(action, meta_action, doc, update_spec)

        # Leave _id, since it's part of the original document
        doc['_id'] = doc_id
852715
def add_documents(self, documents):
        '''
        
        '''
        result, errors = [], []
        for document in documents:
            try:
                result.append(self.add_document(document))
            except RuntimeError as exc:
                errors.append((document, exc))

        return result, errors
455024
def insert(self, docs, *args, **kwargs):
        """"""
        if isinstance(docs, list):
            return self.insert_many(docs, *args, **kwargs)
        else:
            return self.insert_one(docs, *args, **kwargs)
897412
def _index_document(self, document, force=False):
        
        query = text("""
            INSERT INTO dataset_index(vid, title, keywords, doc)
            VALUES(:vid, :title, string_to_array(:keywords, ' '), to_tsvector('english', :doc));
        """)
        self.execute(query, **document)
647410
def _insert(self, sock_info, docs, ordered=True, check_keys=True,
                manipulate=False, write_concern=None, op_id=None,
                bypass_doc_val=False):
        """"""
        if isinstance(docs, collections.Mapping):
            return self._insert_one(
                sock_info, docs, ordered,
                check_keys, manipulate, write_concern, op_id, bypass_doc_val)

        ids = []

        if manipulate:
            def gen():
                """Generator that applies SON manipulators to each document
                and adds _id if necessary.
                """
                _db = self.__database
                for doc in docs:
                    # Apply user-configured SON manipulators. This order of
                    # operations is required for backwards compatibility,
                    # see PYTHON-709.
                    doc = _db._apply_incoming_manipulators(doc, self)
                    if not (isinstance(doc, RawBSONDocument) or '_id' in doc):
                        doc['_id'] = ObjectId()

                    doc = _db._apply_incoming_copying_manipulators(doc, self)
                    ids.append(doc['_id'])
                    yield doc
        else:
            def gen():
                """Generator that only tracks existing _ids."""
                for doc in docs:
                    # Don't inflate RawBSONDocument by touching fields.
                    if not isinstance(doc, RawBSONDocument):
                        ids.append(doc.get('_id'))
                    yield doc

        concern = (write_concern or self.write_concern).document
        acknowledged = concern.get("w") != 0

        command = SON([('insert', self.name),
                       ('ordered', ordered)])
        if concern:
            command['writeConcern'] = concern
        if op_id is None:
            op_id = message._randint()
        if bypass_doc_val and sock_info.max_wire_version >= 4:
            command['bypassDocumentValidation'] = True
        bwc = message._BulkWriteContext(
            self.database.name, command, sock_info, op_id,
            self.database.client._event_listeners)
        if sock_info.max_wire_version > 1 and acknowledged:
            # Batched insert command.
            results = message._do_batched_write_command(
                self.database.name + ".$cmd", message._INSERT, command,
                gen(), check_keys, self.__write_response_codec_options, bwc)
            _check_write_command_response(results)
        else:
            # Legacy batched OP_INSERT.
            message._do_batched_insert(self.__full_name, gen(), check_keys,
                                       acknowledged, concern, not ordered,
                                       self.__write_response_codec_options, bwc)
        return ids
104673
def insert_one(self, document, **kwargs):
        
        self._arctic_lib.check_quota()
        return self._collection.insert_one(document, **kwargs)
1089542
def write(self, values):
        
        # Insert only for docs targeted by the target
        filtered = {_id: value for _id, value in values.items() if _id in self._document_ids}

        if not filtered:
            return

        bulk = self.get_collection().initialize_ordered_bulk_op()
        for _id, value in filtered.items():
            bulk.find({'_id': _id}).upsert() \
                    .update_one({'$set': {self._field: value}})

        bulk.execute()
1042295
def export_items(elastic_url, in_index, out_index, elastic_url_out=None,
                 search_after=False, search_after_value=None, limit=None,
                 copy=False):
    

    if not limit:
        limit = DEFAULT_LIMIT

    if search_after_value:
        search_after_value_timestamp = int(search_after_value[0])
        search_after_value_uuid = search_after_value[1]
        search_after_value = [search_after_value_timestamp, search_after_value_uuid]

    logging.info("Exporting items from %s/%s to %s", elastic_url, in_index, out_index)

    count_res = requests.get('%s/%s/_count' % (elastic_url, in_index))
    try:
        count_res.raise_for_status()
    except requests.exceptions.HTTPError:
        if count_res.status_code == 404:
            logging.error("The index does not exists: %s", in_index)
        else:
            logging.error(count_res.text)
        sys.exit(1)

    logging.info("Total items to copy: %i", count_res.json()['count'])

    # Time to upload the items with the correct mapping
    elastic_in = ElasticSearch(elastic_url, in_index)
    if not copy:
        # Create the correct mapping for the data sources detected from in_index
        ds_mapping = find_mapping(elastic_url, in_index)
    else:
        logging.debug('Using the input index mapping')
        ds_mapping = extract_mapping(elastic_url, in_index)

    if not elastic_url_out:
        elastic_out = ElasticSearch(elastic_url, out_index, mappings=ds_mapping)
    else:
        elastic_out = ElasticSearch(elastic_url_out, out_index, mappings=ds_mapping)

    # Time to just copy from in_index to our_index
    uid_field = find_uuid(elastic_url, in_index)
    backend = find_perceval_backend(elastic_url, in_index)
    if search_after:
        total = elastic_out.bulk_upload(fetch(elastic_in, backend, limit,
                                              search_after_value, scroll=False), uid_field)
    else:
        total = elastic_out.bulk_upload(fetch(elastic_in, backend, limit), uid_field)

    logging.info("Total items copied: %i", total)
74173
def reindex(self, new_index_name: str, identifier_key: str, **kwargs) -> 'ElasticIndex':
        
        if 'url' not in kwargs:
            kwargs['url'] = self.url
        if 'doc_type' not in kwargs:
            kwargs['doc_type'] = self.doc_type
        if 'mapping' not in kwargs:
            kwargs['mapping'] = self.mapping
        new_index = ElasticIndex(new_index_name, **kwargs)

        for results in self.scroll(size=500):
            new_index.bulk(results, identifier_key)
        return new_index
954673
public String addDocumentWithParentId(String indexName, String indexType, Object bean,Object parentId) throws ElasticSearchException{
		return addDocument(indexName, indexType, bean,(Object)null,  parentId,(String)null);
	}
74157
def add_documents(self, docs):
        
        for sent in docs:
            sent = map(self.process_token, sent)
            self._token_count.update(sent)
566629
public String addMapDocument(String indexName , Map bean) throws ElasticSearchException{
		return addMapDocument(  indexName,  _doc,   bean);
	}
420958
def create(index_name, body, force, verbose):
    """"""
    result = current_search_client.indices.create(
        index=index_name,
        body=json.load(body),
        ignore=[400] if force else None,
    )
    if verbose:
        click.echo(json.dumps(result))
932006
def init(self):
        """"""
        if not self.export_enable:
            return None

        self.index='{}-{}'.format(self.index, datetime.utcnow().strftime("%Y.%m.%d"))
        template_body =  {
          "mappings": {
            "glances": {
              "dynamic_templates": [
                {
                  "integers": {
                    "match_mapping_type": "long",
                    "mapping": {
                      "type": "integer"
                    }
                  }
                },
                {
                  "strings": {
                    "match_mapping_type": "string",
                    "mapping": {
                      "type": "text",
                      "fields": {
                        "raw": {
                          "type":  "keyword",
                          "ignore_above": 256
                        }
                      }
                    }
                  }
                }
              ]
            }
          }
        }

        try:
            es = Elasticsearch(hosts=['{}:{}'.format(self.host, self.port)])
        except Exception as e:
            logger.critical("Cannot connect to ElasticSearch server %s:%s (%s)" % (self.host, self.port, e))
            sys.exit(2)
        else:
            logger.info("Connected to the ElasticSearch server %s:%s" % (self.host, self.port))

        try:
            index_count = es.count(index=self.index)['count']
        except Exception as e:
            # Index did not exist, it will be created at the first write
            # Create it...
            es.indices.create(index=self.index,body=template_body)
        else:
            logger.info("The index %s exists and holds %s entries." % (self.index, index_count))

        return es
681534
public String addDateDocuments(String indexName, String indexType, List<?> beans) throws ElasticSearchException{
		return addDocuments(this.indexNameBuilder.getIndexName(indexName),   indexType,     beans);
	}
668020
def _insert_compressed(
        collection_name, docs, check_keys, continue_on_error, opts, ctx):
    """"""
    op_insert, max_bson_size = _insert(
        collection_name, docs, check_keys, continue_on_error, opts)
    rid, msg = _compress(2002, op_insert, ctx)
    return rid, msg, max_bson_size