Compares the extracted texts.

        :param item: The corresponding NewscrawlerItem
        :param article_candidate_list: A list, the list of ArticleCandidate-Objects which have been extracted
        :return: A string, the most likely text
177381
def extract(self, item, article_candidate_list):
        
        list_text = []

        # The minimal number of words a text needs to have
        min_number_words = 15

        # The texts of the article candidates and the respective extractors are saved in a tuple in list_text.
        for article_candidate in article_candidate_list:
            if article_candidate.text != None:
                list_text.append((article_candidate.text, article_candidate.extractor))

        # Remove texts that are shorter than min_number_words.
        for text_tuple in list_text:
            if len(text_tuple[0].split()) < min_number_words:
                list_text.remove(text_tuple)

        # If there is no value in the list, return None.
        if len(list_text) == 0:
            return None

        # If there is only one solution, return it.
        if len(list_text) < 2:
            return list_text[0][0]
        else:

            # If there is more than one solution, do the following:

            # Create a list which holds triple of the score and the two extractors
            list_score = []

            # Compare every text with all other texts at least once
            for a, b, in itertools.combinations(list_text, 2):

                # Create sets from the texts
                set_a = set(a[0].split())
                set_b = set(b[0].split())
                symmetric_difference_a_b = set_a ^ set_b
                intersection_a_b = set_a & set_b

                # Replace 0 with -1 in order to elude division by zero
                if intersection_a_b == 0:
                    intersection_a_b = -1

                # Create the score. It divides the number of words which are not in both texts by the number of words which
                # are in both texts and subtracts the result from 1. The closer to 1 the more similiar they are.
                score = 1 - ((len(symmetric_difference_a_b)) / (2 * len(intersection_a_b)))
                list_score.append((score, a[1], b[1]))

            # Find out which is the highest score
            best_score = max(list_score, key=lambda item: item[0])

            # If one of the solutions is newspaper return it
            if "newspaper" in best_score:
                return (list(filter(lambda x: x[1] == "newspaper", list_text))[0][0])
            else:
                # If not, return the text that is longer

                # A list that holds the extracted texts and their extractors which were most similar
                top_candidates = []
                for tuple in list_text:
                    if tuple[1] == best_score[1] or tuple[1] == best_score[2]:
                        top_candidates.append(tuple)

                if len(top_candidates[0][0]) > len(top_candidates[1][0]):
                    return (top_candidates[0][0])
                else:
                    return (top_candidates[1][0])
574007
def listing_searchable_text(instance):
    
    # get all snapshots
    snapshots = get_snapshots(instance)
    # extract all snapshot values, because we are not interested in the
    # fieldnames (keys)
    values = map(lambda s: s.values(), snapshots)
    # prepare a set of unified catalog data
    catalog_data = set()
    # values to skip
    skip_values = ["None", "true", "True", "false", "False"]
    # internal uid -> title cache
    uid_title_cache = {}

    # helper function to recursively unpack the snapshot values
    def append(value):
        if isinstance(value, (list, tuple)):
            map(append, value)
        elif isinstance(value, (dict)):
            map(append, value.items())
        elif isinstance(value, basestring):
            # convert unicode to UTF8
            if isinstance(value, unicode):
                value = api.safe_unicode(value).encode("utf8")
            # skip single short values
            if len(value) < 2:
                return
            # flush non meaningful values
            if value in skip_values:
                return
            # flush ISO dates
            if re.match(DATE_RX, value):
                return
            # fetch the title
            if re.match(UID_RX, value):
                if value in uid_title_cache:
                    value = uid_title_cache[value]
                else:
                    title_or_id = get_title_or_id_from_uid(value)
                    uid_title_cache[value] = title_or_id
                    value = title_or_id
            catalog_data.add(value)

    # extract all meaningful values
    for value in itertools.chain(values):
        append(value)

    return " ".join(catalog_data)
689907
def list_by_x(self, item=None):
        
        prs = self._processors

        if item is None or item == "cid":  # Default.
            res = [(cid, [prs[cid]]) for cid in sorted(prs.keys())]

        elif item in ("type", "extensions"):
            res = list_by_x(prs.values(), item)
        else:
            raise ValueError("keyword argument 'item' must be one of "
                             "None, 'cid', 'type' and 'extensions' "
                             "but it was '%s'" % item)
        return res
50743
def find(self, item, description='', event_type=''):
        
        # TODO: should be refactored, dumb logic
        if ': ' in item:
            splited = item.split(': ', 1)
            if splited[0] in self.TYPES:
                description = item.split(': ')[1]
                event_type = item.split(': ')[0]
            else:
                description = item
        else:
            if not description:
                description = item

        if event_type:
            found = [x['time'] for x in self.log if re.search(description, x['description'])
                     and x['eventTypeText'] == event_type]
        else:
            found = [x['time'] for x in self.log if re.search(description, x['description'])]

        if len(found):
            return found
        raise exceptions.NotFoundError("Item '{}' is not found with (description='{}', event_type='{}')".
                                       format(item, description, event_type))
1283551
def get_candidate_election(self, election):
        """"""
        return CandidateElection.objects.get(candidate=self, election=election)
916882
def match(self, item):
        
        if 0 and getattr(item, self._name):
            print "%r %r %r %r %r %r" % (self._cmp(float(getattr(item, self._name)), self._value),
                  self._name, self._condition, item.name, getattr(item, self._name), self._value)
        val = getattr(item, self._name) or 0
        if self.not_null and self._value and not val:
            return False
        else:
            return self._cmp(float(val), self._value)
501772
def rank_items(self, userid, user_items, selected_items, recalculate_user=False):
        
        # check if selected_items contains itemids that are not in the model(user_items)
        if max(selected_items) >= user_items.shape[1] or min(selected_items) < 0:
            raise IndexError("Some of selected itemids are not in the model")

        # calculate the relevance scores
        liked_vector = user_items[userid]
        recommendations = liked_vector.dot(self.similarity)

        # remove items that are not in the selected_items
        best = sorted(zip(recommendations.indices, recommendations.data), key=lambda x: -x[1])
        ret = [rec for rec in best if rec[0] in selected_items]

        # returned items should be equal to input selected items
        for itemid in selected_items:
            if itemid not in recommendations.indices:
                ret.append((itemid, -1.0))
        return ret
29477
def match(self, item):
        
        tags = getattr(item, self._name) or []
        if self._exact:
            # Equality check
            return self._value == set(tags)
        else:
            # Is given tag in list?
            return self._value in tags
1215838
def get_reprocessing_candidates(self, since=None):
        
        collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)

        query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS,
                                              unit_of_work.STATE_INVALID,
                                              unit_of_work.STATE_REQUESTED]},
                 unit_of_work.UNIT_OF_WORK_TYPE: unit_of_work.TYPE_MANAGED}

        if since is None:
            cursor = collection.find(query).sort('_id', ASCENDING)
            candidates = [UnitOfWork.from_json(document) for document in cursor]
        else:
            candidates = []
            yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since)
            query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}

            cursor = collection.find(query).sort('_id', ASCENDING)
            for document in cursor:
                uow = UnitOfWork.from_json(document)
                if uow.process_name not in context.process_context:
                    # this is a decommissioned process
                    continue

                time_qualifier = context.process_context[uow.process_name].time_qualifier
                if time_qualifier == QUALIFIER_REAL_TIME:
                    time_qualifier = QUALIFIER_HOURLY
                process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since)

                if process_specific_since <= uow.start_timeperiod:
                    candidates.append(uow)

        if len(candidates) == 0:
            raise LookupError('MongoDB has no UOW reprocessing candidates')
        return candidates
882304
def match(self, item):
        
        val = (getattr(item, self._name) or '').lower()
        result = self._matcher(val) if self._is_regex else self._matcher(val, item)
        if 0:
            log.debug("%r for %r ~ %r, name %r, item %r" % (
                result, val, self._value, self._name, item))
        return result
97639
def find_by_name(self, item_name, items_list, name_list=None):
        
        if not name_list:
            names = [item.name for item in items_list if item]
        else:
            names = name_list
        if item_name in names:
            ind = names.index(item_name)
            return items_list[ind]
        return False
1306776
def get_articles(self, issue=''):
        	
		
		
	soup = get_soup() # get soup of all articles 
	issues = soup.find_all('ul')
		
	# validating and assigning default value for issue
	if not type(issue) is int or issue < 0 :
		issue = 1
	if issue > len(issues):
		issue = len(issues)		
		
	# considering latest article is last element 		
	articles = issues[len(issues)-issue].find_all('a') 
	mArticles = []
	for article in articles:
		mArticle = {}
		mArticle['link'] = article.get('href')[1:]
		mArticle['title'] = article.find('li').contents[0].strip()
		mArticle['author'] = article.find('span').contents[0].encode('utf8')
		mArticles.append(mArticle)
	return mArticles
581332
def find_candidates(document):
    
    nodes_to_score = set()
    should_remove = set()

    for node in document.iter():
        if is_unlikely_node(node):
            logger.debug(
                "We should drop unlikely: %s %r", node.tag, node.attrib)
            should_remove.add(node)
        elif is_bad_link(node):
            logger.debug(
                "We should drop bad link: %s %r", node.tag, node.attrib)
            should_remove.add(node)
        elif node.tag in SCORABLE_TAGS:
            nodes_to_score.add(node)

    return score_candidates(nodes_to_score), should_remove
716703
private static List<UberData> doExtractLinksAndContent(Object item) {

		if (item instanceof EntityModel) {
			return extractLinksAndContent((EntityModel<?>) item);
		}

		if (item instanceof RepresentationModel) {
			return extractLinksAndContent((RepresentationModel<?>) item);
		}

		return extractLinksAndContent(new EntityModel<>(item));
	}
1070548
public boolean filterMatches(AbstractItem item) throws MessageStoreException
  {
    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled())
      SibTr.entry(tc, "filterMatches", item);

    boolean result = true;
    
    // If messages are classified by XD, then check the classification up front.
    // If the index is 0, then there is no classification
    if(classIndex>0)
    {
      // If we're classifying then the classification property must match this
      // filter's classification
      result = false;
      
      //Do not throw exception if Message not available in the store
      boolean throwExceptionIfMessageNotAvailable = false;
      
      // Need to get the classification out of the message
      String keyClassification = ((SIMPMessage)item).getMessageControlClassification(throwExceptionIfMessageNotAvailable);              

      if(keyClassification != null && keyClassification.equalsIgnoreCase(classificationName))
        result = true;
      else
      {
        if(TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
          SibTr.debug(tc, "filter class: " + classificationName + ", message class: " + keyClassification);
      }
    }
    
    // If we still have a potential match, then drive any selector
    // processing through the parent class.
    if(result)
      result = parentFilter.filterMatches(item);

    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled())
      SibTr.exit(tc, "filterMatches", Boolean.valueOf(result));
    
    return result;
  }
1223287
def process_item(self, item, spider):
        

        # Set defaults
        version = 1
        ancestor = 0

        # Search the CurrentVersion table for an old version of the article
        try:
            self.cursor.execute(self.compare_versions, (item['url'],))
        except (pymysql.err.OperationalError, pymysql.ProgrammingError, pymysql.InternalError,
                pymysql.IntegrityError, TypeError) as error:
            self.log.error("Something went wrong in query: %s", error)

        # Save the result of the query. Must be done before the add,
        # otherwise the result will be overwritten in the buffer
        old_version = self.cursor.fetchone()

        if old_version is not None:
            old_version_list = {
                'db_id': old_version[0],
                'local_path': old_version[1],
                'modified_date': old_version[2],
                'download_date': old_version[3],
                'source_domain': old_version[4],
                'url': old_version[5],
                'html_title': old_version[6],
                'ancestor': old_version[7],
                'descendant': old_version[8],
                'version': old_version[9],
                'rss_title': old_version[10], }

            # Update the version number and the ancestor variable for later references
            version = (old_version[9] + 1)
            ancestor = old_version[0]

        # Add the new version of the article to the CurrentVersion table
        current_version_list = {
            'local_path': item['local_path'],
            'modified_date': item['modified_date'],
            'download_date': item['download_date'],
            'source_domain': item['source_domain'],
            'url': item['url'],
            'html_title': item['html_title'],
            'ancestor': ancestor,
            'descendant': 0,
            'version': version,
            'rss_title': item['rss_title'], }

        try:
            self.cursor.execute(self.insert_current, current_version_list)
            self.conn.commit()
            self.log.info("Article inserted into the database.")
        except (pymysql.err.OperationalError, pymysql.ProgrammingError, pymysql.InternalError,
                pymysql.IntegrityError, TypeError) as error:
            self.log.error("Something went wrong in commit: %s", error)

        # Move the old version from the CurrentVersion table to the ArchiveVersions table
        if old_version is not None:
            # Set descendant attribute
            try:
                old_version_list['descendant'] = self.cursor.lastrowid
            except (pymysql.err.OperationalError, pymysql.ProgrammingError, pymysql.InternalError,
                    pymysql.IntegrityError, TypeError) as error:
                self.log.error("Something went wrong in id query: %s", error)

            # Delete the old version of the article from the CurrentVersion table
            try:
                self.cursor.execute(self.delete_from_current, old_version_list['db_id'])
                self.conn.commit()
            except (pymysql.err.OperationalError, pymysql.ProgrammingError, pymysql.InternalError,
                    pymysql.IntegrityError, TypeError) as error:
                self.log.error("Something went wrong in delete: %s", error)

            # Add the old version to the ArchiveVersion table
            try:
                self.cursor.execute(self.insert_archive, old_version_list)
                self.conn.commit()
                self.log.info("Moved old version of an article to the archive.")
            except (pymysql.err.OperationalError, pymysql.ProgrammingError, pymysql.InternalError,
                    pymysql.IntegrityError, TypeError) as error:
                self.log.error("Something went wrong in archive: %s", error)

        return item
193100
def get_source_data_items(self, data_item: DataItem) -> typing.List[DataItem]:
        
        return [DataItem(data_item) for data_item in self._document_model.get_source_data_items(data_item._data_item)] if data_item else None