convert instance to Solr document
505873
def to_solr_doc
      debug "to_solr_doc: creating doc for class: #{self.class.name}, id: #{record_id(self)}"
      doc = Solr::Document.new
      doc.boost = validate_boost(configuration[:boost]) if configuration[:boost]
      
      doc << {:id => solr_id,
              solr_configuration[:type_field] => self.class.name,
              solr_configuration[:primary_key_field] => record_id(self).to_s}

      # iterate through the fields and add them to the document,
      configuration[:solr_fields].each do |field_name, options|
        next if [self.class.primary_key, "type"].include?(field_name.to_s)

        field_boost = options[:boost] || solr_configuration[:default_boost]
        field_type = get_solr_field_type(options[:type])
        solr_name = options[:as] || field_name
        
        value = self.send("#{field_name}_for_solr") rescue nil
        next if value.nil?

        suffix = get_solr_field_type(field_type)
        value = Array(value).map{ |v| ERB::Util.html_escape(v) } # escape each value
        value = value.first if value.size == 1

        field = Solr::Field.new(:name => "#{solr_name}_#{suffix}", :value => value)
        processed_boost = validate_boost(field_boost)
        field.boost = processed_boost
        doc << field
      end
      
      add_dynamic_attributes(doc)
      add_includes(doc)
      add_tags(doc)
      add_space(doc)
      
      debug doc.to_json
      doc
    end
822419
def to_representation(self, field, instance):
        
        self.parent_field = field
        return super(FacetFieldSerializer, self).to_representation(instance)
754129
def to_object(record, attribute_map, stringify_keys = false)
      attributes = {}
      attribute_map.each do |map|
        map = map.inject({}) { |memo,(k,v)| memo[k.to_s] = v; memo} if stringify_keys
        if map["with"]
          as = deep_find(record, map["key"], map["nested_key"])
          values = []
          if as.is_a? Array
            values = as.map { |a| strip_refname( deep_find(a, map["with"]) ) }
          elsif as.is_a? Hash and as[ map["with"] ]
            values = as[ map["with"] ].is_a?(Array) ? as[ map["with"] ].map { |a| strip_refname(a) } : [ strip_refname(as[ map["with"] ]) ]
          end
          attributes[map["field"]] = values
        else
          attributes[map["field"]] = deep_find(record, map["key"], map["nested_key"])
        end
      end
      attributes
    end
1277599
def _element_to_bson(key, value, check_keys, opts):
    """"""
    if not isinstance(key, string_type):
        raise InvalidDocument("documents must have only string keys, "
                              "key was %r" % (key,))
    if check_keys:
        if key.startswith("$"):
            raise InvalidDocument("key %r must not start with '$'" % (key,))
        if "." in key:
            raise InvalidDocument("key %r must not contain '.'" % (key,))

    name = _make_name(key)
    return _name_value_to_bson(name, value, check_keys, opts)
1037971
def to_refs(value):
    """"""
    from mongoframes.frames import Frame, SubFrame

    # Frame
    if isinstance(value, Frame):
        return value._id

    # SubFrame
    elif isinstance(value, SubFrame):
        return to_refs(value._document)

    # Lists
    elif isinstance(value, (list, tuple)):
        return [to_refs(v) for v in value]

    # Dictionaries
    elif isinstance(value, dict):
        return {k: to_refs(v) for k, v in value.items()}

    return value
1185549
def _elements_to_dict(data, position, obj_end, opts, subdocument=None):
    """"""
    if type(opts.document_class) == tuple:
        result = opts.document_class[0](**opts.document_class[1]) if not subdocument else dict()
    else:
        result = opts.document_class() if not subdocument else dict()
    end = obj_end - 1
    while position < end:
        (key, value, position) = _element_to_dict(data, position, obj_end, opts)
        result[key] = value
    return result
172374
def get_instance(page_to_crawl):
    """"""
    global _instances
    if isinstance(page_to_crawl, basestring):
        uri = page_to_crawl
        page_to_crawl = crawlpage.get_instance(uri)
    elif isinstance(page_to_crawl, crawlpage.CrawlPage):
        uri = page_to_crawl.uri
    else:
        raise TypeError(
            "get_instance() expects a parker.CrawlPage "
            "or basestring derivative."
        )

    try:
        instance = _instances[uri]
    except KeyError:
        instance = CrawlModel(page_to_crawl)
        _instances[uri] = instance

    return instance
255231
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'environment_id' in _dict:
            args['environment_id'] = _dict.get('environment_id')
        if 'customer_id' in _dict:
            args['customer_id'] = _dict.get('customer_id')
        if 'document_type' in _dict:
            args['document_type'] = _dict.get('document_type')
        if 'natural_language_query' in _dict:
            args['natural_language_query'] = _dict.get('natural_language_query')
        if 'document_results' in _dict:
            args[
                'document_results'] = LogQueryResponseResultDocuments._from_dict(
                    _dict.get('document_results'))
        if 'created_timestamp' in _dict:
            args['created_timestamp'] = string_to_datetime(
                _dict.get('created_timestamp'))
        if 'client_timestamp' in _dict:
            args['client_timestamp'] = string_to_datetime(
                _dict.get('client_timestamp'))
        if 'query_id' in _dict:
            args['query_id'] = _dict.get('query_id')
        if 'session_token' in _dict:
            args['session_token'] = _dict.get('session_token')
        if 'collection_id' in _dict:
            args['collection_id'] = _dict.get('collection_id')
        if 'display_rank' in _dict:
            args['display_rank'] = _dict.get('display_rank')
        if 'document_id' in _dict:
            args['document_id'] = _dict.get('document_id')
        if 'event_type' in _dict:
            args['event_type'] = _dict.get('event_type')
        if 'result_type' in _dict:
            args['result_type'] = _dict.get('result_type')
        return cls(**args)
108097
def _compile_query(self):
        
        # https://wiki.apache.org/solr/SolrQuerySyntax
        # http://lucene.apache.org/core/2_9_4/queryparsersyntax.html
        query = []

        # filtered_query = self._model_class.row_level_access(self._current_context, self)
        # if filtered_query is not None:
        #     self._solr_query += filtered_query._solr_query
        # print(self._solr_query)
        for key, val, is_escaped in self._solr_query:
            # querying on a linked model by model instance
            # it should be a Model, not a Node!
            if key == 'key':
                key = '_yz_rk'
            elif key == '-key':
                    key = '-_yz_rk'
            elif key[:5] == 'key__':  # to handle key__in etc.
                key = '_yz_rk__' + key[5:]
            elif key[:6] == '-key__':  # to handle key__in etc.
                key = '-_yz_rk__' + key[6:]

            key, val, is_escaped = self._process_query_val(key, val, is_escaped)
            # if it's not one of the expected objects, it should be a string
            # if key == "OR_QRY" then join them with "OR" after escaping & parsing
            if key == 'OR_QRY':
                key = 'NOKEY'
                val = ' OR '.join(
                    ['%s:%s' % self._parse_query_key(*self._process_query_val(k, v, is_escaped)) for
                     k, v in val.items()])
                is_escaped = True
            # __in query is same as OR_QRY but key stays same for all values
            elif key.endswith('__in'):
                if not val:
                    raise ValueError("query value list can not be empty for __in query, "
                                     "please check if it is empty or not before execute filter.")
                key = key[:-4]
                val = ' OR '.join(
                    ['%s:%s' % (key, self._escape_query(v, is_escaped)) for v in val])
                if key.startswith('-'):
                    val = '*:* %s' % val
                key = 'NOKEY'
                is_escaped = True
            # parse the query
            key, val = self._parse_query_key(key, val, is_escaped)

            # as long as not explicitly asked for,
            # we filter out records with deleted flag
            if key == 'deleted':
                self.want_deleted = True
            # convert two underscores to dot notation
            key = key.replace('__', '.')
            # NOKEY means we already combined key partition in to "val"
            if key == 'NOKEY':
                query.append("(%s)" % val)
            else:
                query.append("%s:%s" % (key, val))

        # need to add *:* for negative queries, if
        # query has only one criteria, such as:
        # (-name:Jack) AND -deleted:True
        # this wont work properly, it must be altered as
        # (*:* -name:Jack) AND -deleted:True
        if len(query) == 1:
            q = query[0]
            if q.startswith('-'):
                query[0] = '*:* %s' % q
            if q[:2] == '(-':
                query[0] = '( *:* %s' % q[1:]

        # filter out "deleted" fields if not user explicitly asked for

        # join everything with "AND"
        joined_query = self._QUERY_GLUE.join(query)
        if not self.want_deleted:
            if joined_query:
                joined_query = "(%s) AND -deleted:True" % joined_query
            else:
                joined_query = '-deleted:True'
        elif not joined_query:
            joined_query = '*:*'
        self.compiled_query = joined_query
741030
def get_instance(*args, **kwargs):
    """"""

    inst = KindleCloudReaderAPI(*args, **kwargs)
    try:
      yield inst
    except Exception:
      raise
    finally:
      inst.close()
1130779
def exec(self, **kwargs):
        

        params = self.solr_params()
        logging.info("PARAMS="+str(params))
        results = self.solr.search(**params)
        n_docs = len(results.docs)
        logging.info("Docs found: {}".format(results.hits))

        if self.iterate:
            docs = results.docs
            start = n_docs
            while n_docs >= self.rows:
                logging.info("Iterating; start={}".format(start))
                next_results = self.solr.search(start=start, **params)
                next_docs = next_results.docs
                n_docs = len(next_docs)
                docs += next_docs
                start += self.rows
            results.docs = docs

        fcs = results.facets

        payload = {
            'facet_counts': translate_facet_field(fcs, self.invert_subject_object),
            'pagination': {},
            'numFound': results.hits
        }

        include_raw=self.include_raw
        if include_raw:
            # note: this is not JSON serializable, do not send via REST
            payload['raw'] = results

        # TODO - check if truncated

        logging.info("COMPACT={} INV={}".format(self.use_compact_associations, self.invert_subject_object))
        if self.use_compact_associations:
            payload['compact_associations'] = self.translate_docs_compact(results.docs, field_mapping=self.field_mapping,
                                                                     slim=self.slim, invert_subject_object=self.invert_subject_object,
                                                                     map_identifiers=self.map_identifiers, **kwargs)
        else:
            payload['associations'] = self.translate_docs(results.docs, field_mapping=self.field_mapping, map_identifiers=self.map_identifiers, **kwargs)

        if 'facet_pivot' in fcs:
            payload['facet_pivot'] = fcs['facet_pivot']
        if 'facets' in results.raw_response:
            payload['facets'] = results.raw_response['facets']

        # For solr, we implement this by finding all facets
        # TODO: no need to do 2nd query, see https://wiki.apache.org/solr/SimpleFacetParameters#Parameters
        fetch_objects=self.fetch_objects
        if fetch_objects:
            core_object_field = M.OBJECT
            if self.slim is not None and len(self.slim)>0:
                core_object_field = M.OBJECT_CLOSURE
            object_field = map_field(core_object_field, self.field_mapping)
            if self.invert_subject_object:
                object_field = map_field(M.SUBJECT, self.field_mapping)
            oq_params = params.copy()
            oq_params['fl'] = []
            oq_params['facet.field'] = [object_field]
            oq_params['facet.limit'] = -1
            oq_params['rows'] = 0
            oq_params['facet.mincount'] = 1
            oq_results = self.solr.search(**oq_params)
            ff = oq_results.facets['facet_fields']
            ofl = ff.get(object_field)
            # solr returns facets counts as list, every 2nd element is number, we don't need the numbers here
            payload['objects'] = ofl[0::2]

        fetch_subjects=self.fetch_subjects
        if fetch_subjects:
            core_subject_field = M.SUBJECT
            if self.slim is not None and len(self.slim)>0:
                core_subject_field = M.SUBJECT_CLOSURE
            subject_field = map_field(core_subject_field, self.field_mapping)
            if self.invert_subject_object:
                subject_field = map_field(M.SUBJECT, self.field_mapping)
            oq_params = params.copy()
            oq_params['fl'] = []
            oq_params['facet.field'] = [subject_field]
            oq_params['facet.limit'] = self.max_rows
            oq_params['rows'] = 0
            oq_params['facet.mincount'] = 1
            oq_results = self.solr.search(**oq_params)
            ff = oq_results.facets['facet_fields']
            ofl = ff.get(subject_field)
            # solr returns facets counts as list, every 2nd element is number, we don't need the numbers here
            payload['subjects'] = ofl[0::2]
            if len(payload['subjects']) == self.max_rows:
                payload['is_truncated'] = True

        if self.slim is not None and len(self.slim)>0:
            if 'objects' in payload:
                payload['objects'] = [x for x in payload['objects'] if x in self.slim]
            if 'associations' in payload:
                for a in payload['associations']:
                    a['slim'] = [x for x in a['object_closure'] if x in self.slim]
                    del a['object_closure']

        return payload
597340
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'collection_id' in _dict:
            args['collection_id'] = _dict.get('collection_id')
        if 'name' in _dict:
            args['name'] = _dict.get('name')
        if 'description' in _dict:
            args['description'] = _dict.get('description')
        if 'created' in _dict:
            args['created'] = string_to_datetime(_dict.get('created'))
        if 'updated' in _dict:
            args['updated'] = string_to_datetime(_dict.get('updated'))
        if 'status' in _dict:
            args['status'] = _dict.get('status')
        if 'configuration_id' in _dict:
            args['configuration_id'] = _dict.get('configuration_id')
        if 'language' in _dict:
            args['language'] = _dict.get('language')
        if 'document_counts' in _dict:
            args['document_counts'] = DocumentCounts._from_dict(
                _dict.get('document_counts'))
        if 'disk_usage' in _dict:
            args['disk_usage'] = CollectionDiskUsage._from_dict(
                _dict.get('disk_usage'))
        if 'training_status' in _dict:
            args['training_status'] = TrainingStatus._from_dict(
                _dict.get('training_status'))
        if 'source_crawl' in _dict:
            args['source_crawl'] = SourceStatus._from_dict(
                _dict.get('source_crawl'))
        return cls(**args)
672697
def to_query(self):
        
        query = {}

        for field_instance in self.fields:
            query.update(field_instance.to_query())

        return query
1205434
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'document') and self.document is not None:
            _dict['document'] = self.document._to_dict()
        if hasattr(self, 'model_id') and self.model_id is not None:
            _dict['model_id'] = self.model_id
        if hasattr(self, 'model_version') and self.model_version is not None:
            _dict['model_version'] = self.model_version
        if hasattr(self, 'elements') and self.elements is not None:
            _dict['elements'] = [x._to_dict() for x in self.elements]
        if hasattr(self, 'tables') and self.tables is not None:
            _dict['tables'] = [x._to_dict() for x in self.tables]
        if hasattr(
                self,
                'document_structure') and self.document_structure is not None:
            _dict['document_structure'] = self.document_structure._to_dict()
        if hasattr(self, 'parties') and self.parties is not None:
            _dict['parties'] = [x._to_dict() for x in self.parties]
        if hasattr(self,
                   'effective_dates') and self.effective_dates is not None:
            _dict['effective_dates'] = [
                x._to_dict() for x in self.effective_dates
            ]
        if hasattr(self,
                   'contract_amounts') and self.contract_amounts is not None:
            _dict['contract_amounts'] = [
                x._to_dict() for x in self.contract_amounts
            ]
        if hasattr(self,
                   'termination_dates') and self.termination_dates is not None:
            _dict['termination_dates'] = [
                x._to_dict() for x in self.termination_dates
            ]
        if hasattr(self, 'contract_type') and self.contract_type is not None:
            _dict['contract_type'] = [x._to_dict() for x in self.contract_type]
        return _dict
715147
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'environment_id') and self.environment_id is not None:
            _dict['environment_id'] = self.environment_id
        if hasattr(self, 'customer_id') and self.customer_id is not None:
            _dict['customer_id'] = self.customer_id
        if hasattr(self, 'document_type') and self.document_type is not None:
            _dict['document_type'] = self.document_type
        if hasattr(self, 'natural_language_query'
                  ) and self.natural_language_query is not None:
            _dict['natural_language_query'] = self.natural_language_query
        if hasattr(self,
                   'document_results') and self.document_results is not None:
            _dict['document_results'] = self.document_results._to_dict()
        if hasattr(self,
                   'created_timestamp') and self.created_timestamp is not None:
            _dict['created_timestamp'] = datetime_to_string(
                self.created_timestamp)
        if hasattr(self,
                   'client_timestamp') and self.client_timestamp is not None:
            _dict['client_timestamp'] = datetime_to_string(
                self.client_timestamp)
        if hasattr(self, 'query_id') and self.query_id is not None:
            _dict['query_id'] = self.query_id
        if hasattr(self, 'session_token') and self.session_token is not None:
            _dict['session_token'] = self.session_token
        if hasattr(self, 'collection_id') and self.collection_id is not None:
            _dict['collection_id'] = self.collection_id
        if hasattr(self, 'display_rank') and self.display_rank is not None:
            _dict['display_rank'] = self.display_rank
        if hasattr(self, 'document_id') and self.document_id is not None:
            _dict['document_id'] = self.document_id
        if hasattr(self, 'event_type') and self.event_type is not None:
            _dict['event_type'] = self.event_type
        if hasattr(self, 'result_type') and self.result_type is not None:
            _dict['result_type'] = self.result_type
        return _dict
554687
def get_document_field(instance):
        
        for name, field in instance.searchindex.fields.items():
            if field.document is True:
                return name
16872
def _build_document_converter(cls, session: AppSession):
        ''''''

        if not session.args.convert_links:
            return

        converter = session.factory.new(
            'BatchDocumentConverter',
            session.factory['HTMLParser'],
            session.factory['ElementWalker'],
            session.factory['URLTable'],
            backup=session.args.backup_converted
        )

        return converter