Returns a chunk list retrieved from Syntax Analysis results.

    Args:
      input_text (str): Text to annotate.
      language (:obj:`str`, optional): Language of the text.

    Returns:
      A chunk list. (:obj:`budou.chunk.ChunkList`)
352446
def _get_source_chunks(self, input_text, language=None):
    
    chunks = ChunkList()
    seek = 0
    result = self._get_annotations(input_text, language=language)
    tokens = result['tokens']
    language = result['language']
    for i, token in enumerate(tokens):
      word = token['text']['content']
      begin_offset = token['text']['beginOffset']
      label = token['dependencyEdge']['label']
      pos = token['partOfSpeech']['tag']
      if begin_offset > seek:
        chunks.append(Chunk.space())
        seek = begin_offset
      chunk = Chunk(word, pos, label)
      if chunk.label in _DEPENDENT_LABEL:
        # Determining concatenating direction based on syntax dependency.
        chunk.dependency = i < token['dependencyEdge']['headTokenIndex']
      if chunk.is_punct():
        chunk.dependency = chunk.is_open_punct()
      chunks.append(chunk)
      seek += len(word)
    return chunks, language
1069982
def split_locale(locale: Text) -> Tuple[Text, Optional[Text]]:
    

    items = re.split(r'[_\-]', locale.lower(), 1)

    try:
        return items[0], items[1]
    except IndexError:
        return items[0], None
642550
def _load_sentence_list(self, path):
        

        result = {}

        for entry in textfile.read_separated_lines_generator(path, separator='\t', max_columns=3):
            if self.include_languages is None or entry[1] in self.include_languages:
                result[entry[0]] = entry[1:]

        return result
641230
def render_text(text, language=None):
    
    # Get the filter
    text_filter = SUPPORTED_LANGUAGES.get(language, None)
    if not text_filter:
        raise ImproperlyConfigured("markup filter does not exist: {0}. Valid options are: {1}".format(
            language, ', '.join(list(SUPPORTED_LANGUAGES.keys()))
        ))

    # Convert.
    return text_filter(text)
681094
private List<String> extractWords(@NotNull String text, Language language) {

        List<String> words = textUtils.extractWords(text, language);

        List<String> wordList = words.stream()
                                     .flatMap(word -> {
                                         return textUtils.getNormalForms(word.toLowerCase(), language)
                                                         .stream()
                                                         .filter(element -> textUtils.isNormalWord(element, language));
                                     })
                                     .collect(Collectors.toList());

        return wordList;
    }
670427
def word_tokenize(self, text: str) -> List[str]:
        
        return word_tokenize(text, custom_dict=self.__trie_dict, engine=self.__engine)
80813
def get_bigram_pair_string(self, text):
        
        bigram_pairs = []

        if len(text) <= 2:
            text_without_punctuation = text.translate(self.punctuation_table)
            if len(text_without_punctuation) >= 1:
                text = text_without_punctuation

        document = self.nlp(text)

        if len(text) <= 2:
            bigram_pairs = [
                token.lemma_.lower() for token in document
            ]
        else:
            tokens = [
                token for token in document if token.is_alpha and not token.is_stop
            ]

            if len(tokens) < 2:
                tokens = [
                    token for token in document if token.is_alpha
                ]

            for index in range(1, len(tokens)):
                bigram_pairs.append('{}:{}'.format(
                    tokens[index - 1].pos_,
                    tokens[index].lemma_.lower()
                ))

        if not bigram_pairs:
            bigram_pairs = [
                token.lemma_.lower() for token in document
            ]

        return ' '.join(bigram_pairs)
1188549
def get_language_tool_feedback(sentence):
    """"""
    payload = {'language':'en-US', 'text':sentence}
    try:
        r = requests.post(LT_SERVER, data=payload)
    except requests.exceptions.ConnectionError as e:
        raise requests.exceptions.ConnectionError('''The languagetool server is
        not running.  Try starting it with "ltserver" ''')
    if r.status_code >= 200 and r.status_code < 300:
        return r.json().get('matches', [])
    return []
433019
def extract_features(self, text):
        
        # Feature extractor may take one or two arguments
        try:
            return self.feature_extractor(text, self.train_set)
        except (TypeError, AttributeError):
            return self.feature_extractor(text)
1129214
def parse(self, text):
        '''
        '''
        results = []
        if isinstance(text, str):
            if six.PY2:
                text = unicode(text, 'utf-8')
        self.clean_text = self._normalize_string(text)

        # get addresses
        addresses = set(self._get_addresses(self.clean_text))
        if addresses:
            # append parsed address info
            results = list(map(self._parse_address, addresses))

        return results
1319461
def tag(s, tokenize=True, encoding="utf-8", **kwargs):
    
    tags = []
    for sentence in parse(s, tokenize, True, False, False, False, encoding, **kwargs).split():
        for token in sentence:
            tags.append((token[0], token[1]))
    return tags
892432
def ParseMany(text):
  
  precondition.AssertType(text, Text)

  if compatibility.PY2:
    text = text.encode("utf-8")

  return list(yaml.safe_load_all(text))
1108774
def transform(self, text):
        
        if type(text) is not unicode:
            text = unicode(text, 'utf-8')
        return text
134115
def list_input(*args, **kwargs):
    '''
    
    '''
    list_input = wtforms.TextAreaField(*args, **kwargs)
    list_input.input_type = 'list'
    return list_input
1277217
def segment(text: str, custom_dict: Trie = None) -> List[str]:
    
    if not text or not isinstance(text, str):
        return []

    return list(_multicut(text, custom_dict=custom_dict))
631337
def get_word_list_eng(text):
    
    words, index = [''], 0
    while index < len(text):
        while index < len(text) and ('a' <= text[index] <= 'z' or 'A' <= text[index] <= 'Z'):
            words[-1] += text[index]
            index += 1
        if words[-1]:
            words.append('')
        while index < len(text) and not ('a' <= text[index] <= 'z' or 'A' <= text[index] <= 'Z'):
            if text[index] != ' ':
                words[-1] += text[index]
            index += 1
        if words[-1]:
            words.append('')
    if not words[-1]:
        words.pop()
    return words
677443
def tree(s, token=[WORD, POS, CHUNK, PNP, REL, LEMMA]):
    
    return Text(s, token)