Default tokenizer function that uses jieba tokenizer.

    :param keyword_weight_pair: maximum pair number of the keyword-weight list.
    :return: return keyword-weight list. Example: [('Example',0.4511233019962264),('Hello',0.25548051420382073),...].
173866
def _default_tokenizer_func(content, keyword_weight_pair):
    
    seg_list = jieba.lcut_for_search(content)
    # Extract keyword-weight list by TF-IDF algorithms and by sorted maximum weight
    return jieba.analyse.extract_tags("".join(seg_list), topK=keyword_weight_pair, withWeight=True)
705660
def blkio_weight_device(val, **kwargs):  # pylint: disable=unused-argument
    '''
    
    '''
    val = helpers.map_vals(val, 'Path', 'Weight')
    for idx in range(len(val)):
        try:
            val[idx]['Weight'] = int(val[idx]['Weight'])
        except (TypeError, ValueError):
            raise SaltInvocationError(
                'Weight \'{Weight}\' for path \'{Path}\' is not an '
                'integer'.format(**val[idx])
            )
    return val
625682
def get_strings_from_utterance(tokenized_utterance: List[Token]) -> Dict[str, List[int]]:
    
    string_linking_scores: Dict[str, List[int]] = defaultdict(list)

    for index, token in enumerate(tokenized_utterance):
        for string in ATIS_TRIGGER_DICT.get(token.text.lower(), []):
            string_linking_scores[string].append(index)

    token_bigrams = bigrams([token.text for token in tokenized_utterance])
    for index, token_bigram in enumerate(token_bigrams):
        for string in ATIS_TRIGGER_DICT.get(' '.join(token_bigram).lower(), []):
            string_linking_scores[string].extend([index,
                                                  index + 1])

    trigrams = ngrams([token.text for token in tokenized_utterance], 3)
    for index, trigram in enumerate(trigrams):
        if trigram[0] == 'st':
            natural_language_key = f'st. {trigram[2]}'.lower()
        else:
            natural_language_key = ' '.join(trigram).lower()
        for string in ATIS_TRIGGER_DICT.get(natural_language_key, []):
            string_linking_scores[string].extend([index,
                                                  index + 1,
                                                  index + 2])
    return string_linking_scores
618012
def addKeyword(self, keyword, weight):
        
        assert isinstance(weight, (float, int)), "weight value has to be a positive or negative integer"
        self.topicPage["keywords"].append({"keyword": keyword, "wgt": weight})
273510
public static Pair<String, MultiDimensionalMap<Integer, Integer, String>> stringWithLabels(String sentence,
                                                       TokenizerFactory tokenizerFactory) {
        MultiDimensionalMap<Integer, Integer, String> map = MultiDimensionalMap.newHashBackedMap();
        Tokenizer t = tokenizerFactory.create(sentence);
        List<String> currTokens = new ArrayList<>();
        String currLabel = null;
        String endLabel = null;
        List<Pair<String, List<String>>> tokensWithSameLabel = new ArrayList<>();
        while (t.hasMoreTokens()) {
            String token = t.nextToken();
            if (token.matches(BEGIN_LABEL)) {
                currLabel = token;

                //no labels; add these as NONE and begin the new label
                if (!currTokens.isEmpty()) {
                    tokensWithSameLabel.add(new Pair<>("NONE", (List<String>) new ArrayList<>(currTokens)));
                    currTokens.clear();

                }

            } else if (token.matches(END_LABEL)) {
                if (currLabel == null)
                    throw new IllegalStateException("Found an ending label with no matching begin label");
                endLabel = token;
            } else
                currTokens.add(token);

            if (currLabel != null && endLabel != null) {
                currLabel = currLabel.replaceAll("[<>/]", "");
                endLabel = endLabel.replaceAll("[<>/]", "");
                Preconditions.checkState(!currLabel.isEmpty(), "Current label is empty!");
                Preconditions.checkState(!endLabel.isEmpty(), "End label is empty!");
                Preconditions.checkState(currLabel.equals(endLabel), "Current label begin and end did not match for the parse. Was: %s ending with %s",
                        currLabel, endLabel);

                tokensWithSameLabel.add(new Pair<>(currLabel, (List<String>) new ArrayList<>(currTokens)));
                currTokens.clear();


                //clear out the tokens
                currLabel = null;
                endLabel = null;
            }


        }

        //no labels; add these as NONE and begin the new label
        if (!currTokens.isEmpty()) {
            tokensWithSameLabel.add(new Pair<>("none", (List<String>) new ArrayList<>(currTokens)));
            currTokens.clear();

        }

        //now join the output
        StringBuilder strippedSentence = new StringBuilder();
        for (Pair<String, List<String>> tokensWithLabel : tokensWithSameLabel) {
            String joinedSentence = StringUtils.join(tokensWithLabel.getSecond(), " ");
            //spaces between separate parts of the sentence
            if (!(strippedSentence.length() < 1))
                strippedSentence.append(" ");
            strippedSentence.append(joinedSentence);
            int begin = strippedSentence.toString().indexOf(joinedSentence);
            int end = begin + joinedSentence.length();
            map.put(begin, end, tokensWithLabel.getFirst());
        }


        return new Pair<>(strippedSentence.toString(), map);
    }
660952
def search_bm25(cls, term, weights=None, with_score=False,
                    score_alias='score', explicit_ordering=False):
        """"""
        if not weights:
            rank = SQL('rank')
        elif isinstance(weights, dict):
            weight_args = []
            for field in cls._meta.sorted_fields:
                if isinstance(field, SearchField) and not field.unindexed:
                    weight_args.append(
                        weights.get(field, weights.get(field.name, 1.0)))
            rank = fn.bm25(cls._meta.entity, *weight_args)
        else:
            rank = fn.bm25(cls._meta.entity, *weights)

        selection = ()
        order_by = rank
        if with_score:
            selection = (cls, rank.alias(score_alias))
        if with_score and not explicit_ordering:
            order_by = SQL(score_alias)

        return (cls
                .select(*selection)
                .where(cls.match(FTS5Model.clean_query(term)))
                .order_by(order_by))
459944
def get_token(self, text, start=0):
        
        best_class = best_match = None

        for token_class, match in self.matching_tokens(text):
            if best_match and best_match.end() >= match.end():
                continue
            best_match = match
            best_class = token_class

        return best_class, best_match
1201212
private ClassicCounter<Pair<F, L>> convertSVMStructWeights(ClassicCounter<Integer> weights, Index<F> featureIndex, Index<L> labelIndex) {
    // int numLabels = labelIndex.size();
    int numFeatures = featureIndex.size();
    ClassicCounter<Pair<F, L>> newWeights = new ClassicCounter<Pair<F, L>>();
    for (int i : weights.keySet()) {
      L l = labelIndex.get((i-1) / numFeatures); // integer division on purpose
      F f = featureIndex.get((i-1) % numFeatures);
      double w = weights.getCount(i);
      newWeights.incrementCount(new Pair<F, L>(f, l),w);
    }

    return newWeights;
  }
380036
def _parse_weights(weight_args, default_weight=0.6):
    """"""
    weights_dict = {}
    r_group_weight = default_weight
    for weight_arg in weight_args:
        for weight_assignment in weight_arg.split(','):
            if '=' not in weight_assignment:
                raise ValueError(
                    'Invalid weight assignment: {}'.format(weight_assignment))

            key, value = weight_assignment.split('=', 1)
            value = float(value)
            if key == 'R':
                r_group_weight = value
            elif key == '*':
                default_weight = value
            elif hasattr(Atom, key):
                weights_dict[Atom(key)] = value
            else:
                raise ValueError('Invalid element: {}'.format(key))

    return weights_dict, r_group_weight, default_weight
353179
def _get_labels_and_weights(self, content, comments):
        
        # extract content and comments
        if 'content' in self.to_extract and 'comments' in self.to_extract:
            labels = np.logical_or(content[0], comments[0]).astype(int)
            weights = content[1],
        # extract content only
        elif 'content' in self.to_extract:
            labels = content[0]
            weights = content[1]
        # extract comments only
        else:
            labels = comments[0]
            weights = comments[1]
        if self.max_block_weight is None:
            weights = np.minimum(weights, self.max_block_weight)

        return labels, weights
61144
def _tag_most_likely(examples):
    
    tokenized_examples = [_tokenize_by_character_class(example) for example in examples]

    # We currently need the tokenized_examples to all have the same length, so drop instances that have a length
    # that does not equal the mode of lengths within tokenized_examples
    token_lengths = [len(e) for e in tokenized_examples]
    token_lengths_mode = _mode(token_lengths)
    tokenized_examples = [example for example in tokenized_examples if len(example) == token_lengths_mode]

    # Now, we iterate through the tokens, assigning date elements based on their likelihood. In cases where
    # the assignments are unlikely for all date elements, assign filler.
    most_likely = []
    for token_index in range(0, token_lengths_mode):
        tokens = [token[token_index] for token in tokenized_examples]
        probabilities = _percent_match(DATE_ELEMENTS, tokens)
        max_prob = max(probabilities)
        if max_prob < 0.5:
            most_likely.append(Filler(_mode(tokens)))
        else:
            if probabilities.count(max_prob) == 1:
                most_likely.append(DATE_ELEMENTS[probabilities.index(max_prob)])
            else:
                choices = []
                for index, prob in enumerate(probabilities):
                    if prob == max_prob:
                        choices.append(DATE_ELEMENTS[index])
                most_likely.append(_most_restrictive(choices))

    return most_likely
958462
def get_sugg(self, keyword):
        
        url = 'http://w.sugg.sogou.com/sugg/ajaj_json.jsp?key={}&type=wxpub&pr=web'.format(
            quote(keyword.encode('utf-8')))
        r = requests.get(url)
        if not r.ok:
            raise WechatSogouRequestsException('get_sugg', r)

        sugg = re.findall(u'\["' + keyword + '",(.*?),\["', r.text)[0]
        return json.loads(sugg)
1051570
def get_keywords(lexer):
    
    if not hasattr(lexer, 'tokens'):
        return []
    if 'keywords' in lexer.tokens:
        try:
            return lexer.tokens['keywords'][0][0].words
        except:
            pass
    keywords = []
    for vals in lexer.tokens.values():
        for val in vals:
            try:
                if isinstance(val[0], words):
                    keywords.extend(val[0].words)
                else:
                    ini_val = val[0]
                    if ')\\b' in val[0] or ')(\\s+)' in val[0]:
                        val = re.sub(r'\\.', '', val[0])
                        val = re.sub(r'[^0-9a-zA-Z|]+', '', val)
                        if '|' in ini_val:
                            keywords.extend(val.split('|'))
                        else:
                            keywords.append(val)
            except Exception:
                continue
    return keywords
633922
def getWeights(self, fromName, toName):
        
        for connection in self.connections:
            if connection.fromLayer.name == fromName and \
               connection.toLayer.name == toName:
                return connection.weight
        raise NetworkError('Connection was not found.', (fromName, toName))
1103763
def _weight_drop(module, weights, dropout):
    

    for name_w in weights:
        w = getattr(module, name_w)
        del module._parameters[name_w]
        module.register_parameter(name_w + '_raw', Parameter(w))

    original_module_forward = module.forward

    def forward(*args, **kwargs):
        for name_w in weights:
            raw_w = getattr(module, name_w + '_raw')
            w = torch.nn.functional.dropout(raw_w, p=dropout, training=module.training)
            setattr(module, name_w, w)

        return original_module_forward(*args)

    setattr(module, 'forward', forward)
622692
def total_weight(self):
        
        weights = self._raw_weights()
        if weights.shape[1] == 0:
            return 0.0
        elif weights.shape[1] < self._ntaps:
            return np.sum(np.mean(weights, axis=1))
        else:
            return self._filter_coeffs.dot(np.sum(weights, axis=0))
1052125
def edge_weight(self, edge):
        
        return self.get_edge_properties( edge ).setdefault( self.WEIGHT_ATTRIBUTE_NAME, self.DEFAULT_WEIGHT )