Parameters
        ----------
        ngram, str or unicode, string to search for

        Returns
        -------
        pd.DataFrame, {self._parsed_col: <matching texts>, self._category_col: <corresponding categories>, ...}
350439
def search(self, ngram):
        '''
        

        '''
        mask = self._document_index_mask(ngram)
        return self._df[mask]
147117
def get_ngrams(path):
    """"""
    with open(path, encoding='utf-8') as fh:
        ngrams = [ngram.strip() for ngram in fh.readlines()]
    return ngrams
941235
def ngrams(self, with_counts=False):
        
        query = "SELECT "
        for i in reversed(range(self.cardinality)):
            if i != 0:
                query += "word_{0}, ".format(i)
            elif i == 0:
                query += "word"

        if with_counts:
            query += ", count"

        query += " FROM _{0}_gram;".format(self.cardinality)

        result = self.execute_sql(query)
        for row in result:
            yield tuple(row)
999199
def get_info(df, verbose = None,max_cols = None, memory_usage = None, null_counts = None):
    
    assert type(df) is pd.DataFrame
    buffer = io.StringIO()
    df.info(verbose, buffer, max_cols, memory_usage, null_counts)
    return buffer.getvalue()
1196799
def get_texts(self):
        '''
        
        '''
        if sys.version_info[0] == 2:
            return self._df[self._parsed_col]
        return self._df[self._parsed_col].apply(str)
244214
def parse_columns(lines):
    
    data = []
    index = []
    for line in lines:
        line = line.rstrip()
        if line.startswith("#"):
            tmp = __parse_entry(line)
            data.append(tmp[1])
            index.append(tmp[0])

    return DataFrame(data, index=index, columns=['description'])
258540
def create_ngram_set(input_list, ngram_value=2):
    
    return set(zip(*[input_list[i:] for i in range(ngram_value)]))
394545
def _generate_substrings(self, ngram, size):
        
        text = Text(ngram, self._tokenizer)
        substrings = []
        for sub_size, ngrams in text.get_ngrams(1, size-1):
            for sub_ngram, count in ngrams.items():
                substrings.extend([sub_ngram] * count)
        return substrings
1125759
def load_geo_adwords(filename='AdWords API Location Criteria 2017-06-26.csv.gz'):
    
    df = pd.read_csv(filename, header=0, index_col=0, low_memory=False)
    df.columns = [c.replace(' ', '_').lower() for c in df.columns]
    canonical = pd.DataFrame([list(row) for row in df.canonical_name.str.split(',').values])

    def cleaner(row):
        cleaned = pd.np.array(
            [s for i, s in enumerate(row.values) if s not in ('Downtown', None) and (i > 3 or row[i + 1] != s)])
        if len(cleaned) == 2:
            cleaned = [cleaned[0], None, cleaned[1], None, None]
        else:
            cleaned = list(cleaned) + [None] * (5 - len(cleaned))
        if not pd.np.all(pd.np.array(row.values)[:3] == pd.np.array(cleaned)[:3]):
            logger.info('{} => {}'.format(row.values, cleaned))
        return list(cleaned)

    cleancanon = canonical.apply(cleaner, axis=1)
    cleancanon.columns = 'city region country extra extra2'.split()
    df['region'] = cleancanon.region
    df['country'] = cleancanon.country
    return df
976616
def _load_ngram(name):
    
    module = importlib.import_module('lantern.analysis.english_ngrams.{}'.format(name))
    return getattr(module, name)
137796
def _render_corpus_row(self, label, ngrams):
        """"""
        row = ('<tr>\n<td>{label}</td>\n<td>{first}</td>\n<td>{only}</td>\n'
               '<td>{last}</td>\n</tr>')
        cell_data = {'label': label}
        for period in ('first', 'only', 'last'):
            cell_data[period] = ', '.join(ngrams[label][period])
        return row.format(**cell_data)
339848
def _getTextType(self, lineData, column):
        
        if lineData is None:
            return ' '  # default is code

        textTypeMap = lineData[1]
        if column >= len(textTypeMap):  # probably, not actual data, not updated yet
            return ' '

        return textTypeMap[column]
1372778
private static void ngramOnCharacter(String data, int[] gramSizes, Collection<String> list) {
    	data = data.replaceAll("\\s+", "");
    	for (int j = 0; j < gramSizes.length; j++) {
            int len = gramSizes[j];
            if (len <= 0 || len > data.length())
                continue;
            for (int i = 0; i < data.length() - len; i++) {
                list.add(data.substring(i, i + len));
            }
        }
    }
1189612
def ngram_diff(args, parser):
    """"""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    catalogue = utils.get_catalogue(args)
    tokenizer = utils.get_tokenizer(args)
    store.validate(corpus, catalogue)
    if args.asymmetric:
        store.diff_asymmetric(catalogue, args.asymmetric, tokenizer,
                              sys.stdout)
    else:
        store.diff(catalogue, tokenizer, sys.stdout)
544766
def remove_ngram(self, ngram):
        
        query = "DELETE FROM _{0}_gram".format(len(ngram))
        query += self._build_where_clause(ngram)
        query += ";"
        self.execute_sql(query)
173342
def series2cat(df:DataFrame, *col_names):
    ""
    for c in listify(col_names): df[c] = df[c].astype('category').cat.as_ordered()
1366343
def _get_word_ngrams(n, sentences):
    
    assert len(sentences) > 0
    assert n > 0

    words = _split_into_words(sentences)
    return _get_ngrams(n, words)