Return a .kibana index-pattern doc_type
13164
def field_cache_to_index_pattern(self, field_cache):
        """"""
        mapping_dict = {}
        mapping_dict['customFormats'] = "{}"
        mapping_dict['title'] = self.index_pattern
        # now post the data into .kibana
        mapping_dict['fields'] = json.dumps(field_cache, separators=(',', ':'))
        # in order to post, we need to create the post string
        mapping_str = json.dumps(mapping_dict, separators=(',', ':'))
        return mapping_str
719700
def _find_matching_collections_externally(collections, record):
    
    index, doc_type = RecordIndexer().record_to_index(record)
    body = {"doc": record.dumps()}
    results = current_search_client.percolate(
        index=index,
        doc_type=doc_type,
        allow_no_indices=True,
        ignore_unavailable=True,
        body=body
    )
    prefix_len = len('collection-')
    for match in results['matches']:
        collection_name = match['_id']
        if collection_name.startswith('collection-'):
            name = collection_name[prefix_len:]
            if name in collections:
                yield collections[name]['ancestors']
    raise StopIteration
1255351
def doc_id(self):
        
        if self.id:
            return '%s_%s' % (self.doc_type, self.id.lower())
        return self._hashed_key
756281
def migrate(self, mapping, index, doc_type):
        
        old_mapping = self.get_doctype(index, doc_type)
        #case missing
        if not old_mapping:
            self.connection.indices.put_mapping(doc_type=doc_type, mapping=mapping, indices=index)
            return mapping
            # we need to calculate the diff
        mapping_diff = old_mapping.get_diff(mapping)
        if not mapping_diff:
            return None
        from pprint import pprint

        pprint(mapping_diff.as_dict())
        mapping_diff.connection = old_mapping.connection
        mapping_diff.save()
74173
def reindex(self, new_index_name: str, identifier_key: str, **kwargs) -> 'ElasticIndex':
        
        if 'url' not in kwargs:
            kwargs['url'] = self.url
        if 'doc_type' not in kwargs:
            kwargs['doc_type'] = self.doc_type
        if 'mapping' not in kwargs:
            kwargs['mapping'] = self.mapping
        new_index = ElasticIndex(new_index_name, **kwargs)

        for results in self.scroll(size=500):
            new_index.bulk(results, identifier_key)
        return new_index
68801
def mapping_get(index, doc_type, hosts=None, profile=None):
    '''
    
    '''
    es = _get_instance(hosts, profile)

    try:
        return es.indices.get_mapping(index=index, doc_type=doc_type)
    except elasticsearch.exceptions.NotFoundError:
        return None
    except elasticsearch.TransportError as e:
        raise CommandExecutionError("Cannot retrieve mapping {0}, server returned code {1} with message {2}".format(index, e.status_code, e.error))
744961
def index_data(self, data, index_name, doc_type):
        """"""

        print 'ELS Stub Indexer getting called...'
        print '%s %s %s %s' % (self, data, index_name, doc_type)
127952
public String getDocument(String indexName, String indexType,String documentId) throws ElasticSearchException{
		return getDocument(indexName, indexType,documentId,(Map<String,Object>)null);
	}
687854
def get_index_mappings(self, index):
        """"""
        fields_arr = []
        for (key, val) in iteritems(index):
            # self.pr_dbg("\tdoc_type: %s" % key)
            doc_mapping = self.get_doc_type_mappings(index[key])
            # self.pr_dbg("\tdoc_mapping: %s" % doc_mapping)
            if doc_mapping is None:
                return None
            # keep adding to the fields array
            fields_arr.extend(doc_mapping)
        return fields_arr
1158352
def get(self, index, doc_type, id, fields=None, model=None, **query_params):
        
        path = make_path(index, doc_type, id)
        if fields is not None:
            query_params["fields"] = ",".join(fields)
        model = model or self.model
        return model(self, self._send_request('GET', path, params=query_params))
349334
def format_index(prefix, timestamp, sep='-'):
        
        tz_info = tz.tzutc()

        # ex. logstash-other-2017.05.09
        return "{prefix}{sep}{date}".format(
            prefix=prefix, sep=sep, date=datetime.fromtimestamp(timestamp, tz=tz_info).strftime('%Y.%m.%d'))
735446
def get_document_models():
    """"""
    mappings = {}
    for i in get_index_names():
        for m in get_index_models(i):
            key = "%s.%s" % (i, m._meta.model_name)
            mappings[key] = m
    return mappings
201303
def _index(self, model):
        '''
        
        '''
        doc_type = model
        if not isinstance(model, str):
            doc_type = model.__table__.name

        index_name = doc_type
        if hasattr(model, "__msearch_index__"):
            index_name = model.__msearch_index__

        if doc_type not in self._indexs:
            self._indexs[doc_type] = Index(self._client, index_name, doc_type)
        return self._indexs[doc_type]
264310
def get_default_config(self):
        
        config = super(ElasticSearchCollector, self).get_default_config()
        config.update({
            'host':           '127.0.0.1',
            'port':           9200,
            'user':           '',
            'password':       '',
            'instances':      [],
            'scheme':         'http',
            'path':           'elasticsearch',
            'stats':          ['jvm', 'thread_pool', 'indices'],
            'logstash_mode': False,
            'cluster':       False,
        })
        return config
1310989
def mget(self, ids, index=None, doc_type=None, **query_params):
        
        if not ids:
            return []

        body = []
        for value in ids:
            if isinstance(value, tuple):
                if len(value) == 3:
                    a, b, c = value
                    body.append({"_index": a,
                                 "_type": b,
                                 "_id": c})
                elif len(value) == 4:
                    a, b, c, d = value
                    body.append({"_index": a,
                                 "_type": b,
                                 "_id": c,
                                 "fields": d})

            else:
                if index is None:
                    raise InvalidQuery("index value is required for id")
                if doc_type is None:
                    raise InvalidQuery("doc_type value is required for id")
                body.append({"_index": index,
                             "_type": doc_type,
                             "_id": value})

        results = self._send_request('GET', "/_mget", body={'docs': body},
                                     params=query_params)
        if 'docs' in results:
            model = self.model
            return [model(self, item) for item in results['docs']]
        return []
14958
def get_doc_type_mappings(self, doc_type):
        """"""
        doc_fields_arr = []
        found_score = False
        for (key, val) in iteritems(doc_type):
            # self.pr_dbg("\t\tfield: %s" % key)
            # self.pr_dbg("\tval: %s" % val)
            add_it = False
            retdict = {}
            # _ are system
            if not key.startswith('_'):
                if 'mapping' not in doc_type[key]:
                    self.pr_err("No mapping in doc_type[%s]" % key)
                    return None
                if key in doc_type[key]['mapping']:
                    subkey_name = key
                else:
                    subkey_name = re.sub('.*\.', '', key)
                if subkey_name not in doc_type[key]['mapping']:
                    self.pr_err(
                        "Couldn't find subkey " +
                        "doc_type[%s]['mapping'][%s]" % (key, subkey_name))
                    return None
                # self.pr_dbg("\t\tsubkey_name: %s" % subkey_name)
                retdict = self.get_field_mappings(
                    doc_type[key]['mapping'][subkey_name])
                add_it = True
            # system mappings don't list a type,
            # but kibana makes them all strings
            if key in self.sys_mappings:
                retdict['analyzed'] = False
                retdict['indexed'] = False
                if key == '_source':
                    retdict = self.get_field_mappings(
                        doc_type[key]['mapping'][key])
                    retdict['type'] = "_source"
                elif key == '_score':
                    retdict['type'] = "number"
                elif 'type' not in retdict:
                    retdict['type'] = "string"
                add_it = True
            if add_it:
                retdict['name'] = key
                retdict['count'] = 0  # always init to 0
                retdict['scripted'] = False  # I haven't observed a True yet
                if not self.check_mapping(retdict):
                    self.pr_err("Error, invalid mapping")
                    return None
                # the fields element is an escaped array of json
                # make the array here, after all collected, then escape it
                doc_fields_arr.append(retdict)
        if not found_score:
            doc_fields_arr.append(
                {"name": "_score",
                 "type": "number",
                 "count": 0,
                 "scripted": False,
                 "indexed": False,
                 "analyzed": False,
                 "doc_values": False})
        return doc_fields_arr
783438
def index():
    """"""
    # Prepare mappings from configuration.
    mapping = cfg['UNAPI_FORMAT_MAPPING']
    names = {mapping[k]: k for k in mapping}
    # Clean request arguments.
    identifier = request.values.get('id', type=int)
    format_ = request.values.get('format')
    format_ = mapping.get(format_, format_)
    # Redirect to correct record format.
    if identifier and format_:
        return redirect(url_for('record.metadata', recid=identifier,
                                of=format_))

    formats = [{
        'name': names[of],
        'type': values['content_type'],
        'docs': values.get('url'),
    } for of, values in iteritems(output_formats) if of in names]

    response = make_response(render_template(
        'unapi/index.xml', identifier=identifier, formats=formats))
    response.headers['Content-Type'] = 'application/xml'
    return response