Given a word, get the subwords and their indicies.
15284
def get_subwords(self, word, on_unicode_error='strict'):
        
        pair = self.f.getSubwords(word, on_unicode_error)
        return pair[0], np.array(pair[1])
403155
def get_numwords():
    """"""
    numwords = {'and': (1, 0), 'a': (1, 1), 'an': (1, 1)}

    for idx, word in enumerate(UNITS):
        numwords[word] = (1, idx)
    for idx, word in enumerate(TENS):
        numwords[word] = (1, idx * 10)
    for idx, word in enumerate(SCALES):
        numwords[word] = (10 ** (idx * 3 or 2), 0)

    all_numbers = ur'|'.join(ur'\b%s\b' % i for i in numwords.keys() if i)

    return all_numbers, numwords
163008
def parse(s):
    """"""
    if s.startswith(b'('):
        s = s[1:]
        name, s = parse_term(s)
        values = [name]
        while not s.startswith(b')'):
            value, s = parse(s)
            values.append(value)
        return values, s[1:]

    return parse_term(s)
375720
def parseSearchTerm(term):
    
    terms = []
    keywords = {}
    for word in term.split():
        if word.count(':') == 1:
            k, v = word.split(u':')
            if k and v:
                keywords[k] = v
            elif k or v:
                terms.append(k or v)
        else:
            terms.append(word)
    term = u' '.join(terms)
    if keywords:
        return term, keywords
    return term, None
299129
def word_matches(s1, s2, n=3):
    
    return __matches(s1, s2, word_ngrams, n=n)
217097
def parseWord(word):
    
    mapping = {'yes': True, 'true': True, 'no': False, 'false': False}
    _, key, value = word.split('=', 2)
    try:
        value = int(value)
    except ValueError:
        value = mapping.get(value, value)
    return (key, value)
857979
def get_known_words(self, lang):
        """"""
        words = []
        for topic in self.user_data.language_data[lang]['skills']:
            if topic['learned']:
                words += topic['words']
        return set(words)
856966
def _get_subword_units(token, gram):
    
    if token == '</s>':  # special token for padding purpose.
        return [token]
    t = '#' + token + '#'
    return [t[i:i + gram] for i in range(0, len(t) - gram + 1)]
702394
private int calcCountSubstringsWords(String str, HashSet<String> words) {
    int wordCount = 0;
    int N = str.length();
    for (int i = 0; i < N - 1; i++)
      for (int j = i + 2; j < N + 1; j++) {
        if (words.contains(str.substring(i, j)))
          wordCount += 1;
      }
    return wordCount;
  }
1218187
private Collection<WordSimilarity> parse(File word353file) {

        Collection<WordSimilarity> pairs = new LinkedList<WordSimilarity>();
                
        try {
            BufferedReader br = new BufferedReader(new FileReader(word353file));
            // skip the first line
            br.readLine();
            for (String line = null; (line = br.readLine()) != null; ) {
                
                // skip comments and blank lines
                if (line.startsWith("#") || line.length() == 0) {
                    continue;
                }

                String[] wordsAndNum = line.split("\\s+");
                if (wordsAndNum.length != 3) {
                    throw new Error("Unexpected line formatting: " + line);
                }
                pairs.add(new SimpleWordSimilarity(
                          wordsAndNum[0], wordsAndNum[1], 
                          Double.parseDouble(wordsAndNum[2])));
            }
        } catch (IOException ioe) {
            // rethrow as an IOE is fatal evaluation
            throw new IOError(ioe);
        }
            
        return pairs;
    }
97954
def parse(s, subs):
    """"""
    if len(subs) == 0:
        return []

    points = []
    requests = _tokenize_request(s)

    if len(requests) == 1 and requests[0].type_ == _Request.Type.OFFSET:
        return _offset_subtitles(requests[0], subs)
    return _sync_subtitles(requests, subs)
170903
def getanymentions(idf, anidfobject):
    """"""
    name = anidfobject.obj[1]
    foundobjs = []
    keys = idfobjectkeys(idf)
    idfkeyobjects = [idf.idfobjects[key.upper()] for key in keys]
    for idfobjects in idfkeyobjects:
        for idfobject in idfobjects:
            if name.upper() in [item.upper() 
                                    for item in idfobject.obj 
                                        if isinstance(item, basestring)]:
                foundobjs.append(idfobject)
    return foundobjs
558350
def join(self, words):
        """"""
        chars = list(u"".join(words))
        start = True

        for i in xrange(len(chars)):
            char = chars[i]
            if char.isalpha():
                if start:
                    chars[i] = char.upper()
                else:
                    chars[i] = char.lower()

                start = False
            else:
                if i > 2 and chars[i - 1] in ".?!" and char.isspace():
                    start = True

        return u"".join(chars)
1276957
def get_substituted_contents(contents, substitutions):
    
    result = contents
    for sub in substitutions:
        result = sub.apply_and_get_result(result)
    return result
394494
def sentences(self):
        """"""
        prevp = 0
        prevs = 0
        sentence = [];
        sentence_id = ""
        for word, id, pos, lemma in iter(self):
            try:
                doc_id, ptype, p, s, w = re.findall('([\w\d-]+)\.(p|head)\.(\d+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if ((p != prevp) or (s != prevs)) and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.' + ptype + '.' + str(p) + '.s.' + str(s)
                prevp = p
            except IndexError:
                doc_id, s, w = re.findall('([\w\d-]+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if s != prevs and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.s.' + str(s)
            sentence.append( (word,id,pos,lemma) )
            prevs = s
        if sentence:
            yield sentence_id, sentence
808494
def filter_stopwords(phrase):
    """"""
    if not isinstance(phrase, list):
        phrase = phrase.split()

    stopwords = ['the', 'a', 'in', 'to']
    return [word.lower() for word in phrase if word.lower() not in stopwords]
398314
def get_word_from_offset(self, offset):
        """"""

        if offset+2 > len(self.__data__):
            return None

        return self.get_word_from_data(self.__data__[offset:offset+2], 0)