Parse a Tweet without generating HTML.
355927
def _text(self, text):
        ''''''
        URL_REGEX.sub(self._parse_urls, text)
        USERNAME_REGEX.sub(self._parse_users, text)
        LIST_REGEX.sub(self._parse_lists, text)
        HASHTAG_REGEX.sub(self._parse_tags, text)
        return None
469107
def _parse_content(self, text):
        '''
        '''
        try:
            return json.loads(text)
        except ValueError:
            raise exc.UnexpectedlyNotJSON(
                "The resource at {.uri} wasn't valid JSON", self)
557479
def _parse(self):
        """"""
        while self._cur_token['type'] in (TT.ws, TT.lbreak):
            self._skip_whitespace()
            self._skip_newlines()

        self._data = self._parse_value()

        return self._data
1039436
def parse(contents, tokens=None):
    """"""
    # Shortcut for users who are interested in tokens
    if tokens is None:
        tokens = [t for t in tokenize(contents)]

    token_index, body = _ast_worker(tokens, len(tokens), 0, None)

    assert token_index == len(tokens)
    assert body.arguments == []

    return ToplevelBody(statements=body.statements)
136655
def cleanupPunct( tweet ):
        
        tweet = ''.join( map( lambda c: (unicodedata.name(c).split()[0] in [u'TAMIL',u'LATIN']) and c or u' ', tweet) )
        return tweet
738308
def parse_text(self,text):
        """"""
        try:
            return self.rProgram.ignore(cStyleComment).parseString(text, parseAll=True)
        except SemanticException as err:
            print(err)
            exit(3)
        except ParseException as err:
            print(err)
            exit(3)
1179910
def parseStr(self, html):
        '''
            
        '''
        self.reset()

        if isinstance(html, bytes):
            self.feed(html.decode(self.encoding))
        else:
            self.feed(html)
555732
def process_text(self, t:str, tok:BaseTokenizer) -> List[str]:
        ""
        for rule in self.pre_rules: t = rule(t)
        toks = tok.tokenizer(t)
        for rule in self.post_rules: toks = rule(toks)
        return toks
41595
def cat_tweets(filename='all_tweets.json.gz', path=DATA_PATH, ext='.json', save_tmp=False, verbosity=1, numtweets=10000000, ignore_suspicious=True):
    """"""
    log.info('Finding {} files in {}...'.format(ext, path))
    meta_files = find_files(path=path, ext=ext)
    meta_files = [meta for meta in meta_files
                  if re.match(r'^[-#@a-z ]*201[5-6]-[0-9]{2}-[0-9]{2}.*', meta['name'])]
    #  '\s[0-9]{2}[:][0-9]{2}[:][0-9]{2}[.][0-9]+[.]json(.gz)?$', meta['name'])]
    log.info('Found {} files that look like tweetget dumps.'.format(len(meta_files)))
    print([mf['name'] for mf in meta_files])
    total_size = sum([meta['size'] for meta in meta_files])
    if verbosity > 0:
        pbar = progressbar.ProgressBar(maxval=(total_size + 1.) / 1e6)
        pbar.start()
    else:
        pbar = None

    loaded_size = 0
    df_all = pd.DataFrame()
    for meta in meta_files:
        with (gzip.open(meta['path']) if ext.endswith('.gz') else open(meta['path'])) as fin:
            js = pd.json.load(fin)
        if not len(js):
            continue
        df = pd.io.json.json_normalize(js)
        # json entries were dumped in reverse time order (most recent first)
        df.drop_duplicates(['id'], keep='first', inplace=True)
        df.set_index('id', drop=True, inplace=True)
        if 'geo.coordinates' in df.columns:
            latlon = np.array([(ll[0], ll[1]) if isinstance(ll, list) else (np.nan, np.nan) for ll in df['geo.coordinates']])
            for i, ll in enumerate(latlon):
                try:
                    latlon[i] = float(ll[0]), float(ll[1])
                except ValueError:
                    latlon[i] = np.nan, np.nan
            ll = list(zip(*latlon))
            df['lat'] = ll[0]
            df['lon'] = ll[1]
            df_all = df_all.append(df)
        else:
            df['lat'] = np.nan * np.ones(len(df))
            df['lon'] = np.nan * np.ones(len(df))
            if ignore_suspicious:
                log.info('\nOddly the DataFrame in {} didnt have a geo.coordinates column.'.format(meta['path']))
                log.warn('\nSkipping {} suspicious tweets.'.format(len(df)))
            else:
                df_all = df_all.append(df)
        # this would be a good time to incrementally save these rows to disc
        del df
        loaded_size += meta['size']
        if len(df_all) >= numtweets:
            # FIXME use average rate of 400 tweets/MB to calculate better progressbar size at start
            break
        if save_tmp:
            save_tweets(df_all, path=path, filename='tmp.csv')
        if pbar:
            pbar.update(loaded_size / 1e6)
    print(len(df_all))
    bigger_better_cols = [c for c in df_all.columns if c.endswith('_at') or '_count' in c]
    print(bigger_better_cols)
    print(all(c in df_all.columns for c in bigger_better_cols))
    df_all = df_all.sort_values(by=bigger_better_cols, inplace=False)
    hashable_cols = [c for c in df_all.columns if df_all[c].dtype not in (list,)]
    df_all = df_all.drop_duplicates(subset=hashable_cols, inplace=False)
    if pbar:
        pbar.finish()
    log.info('Loaded {} unique tweets.'.format(len(df_all)))
    return df_all
787727
def __parse_stream(self, stream, parse_line):
        """"""

        if not stream:
            raise InvalidFormatError(cause='stream cannot be empty or None')

        nline = 0
        lines = stream.split('\n')

        for line in lines:
            nline += 1

            # Ignore blank lines and comments
            m = re.match(self.LINES_TO_IGNORE_REGEX, line, re.UNICODE)
            if m:
                continue

            m = re.match(self.VALID_LINE_REGEX, line, re.UNICODE)
            if not m:
                cause = "line %s: invalid format" % str(nline)
                raise InvalidFormatError(cause=cause)

            try:
                result = parse_line(m.group(1), m.group(2))
                yield result
            except InvalidFormatError as e:
                cause = "line %s: %s" % (str(nline), e)
                raise InvalidFormatError(cause=cause)
767665
def idf2txt(txt):
    """"""
    astr = nocomment(txt)
    objs = astr.split(';')
    objs = [obj.split(',') for obj in objs]
    objs = [[line.strip() for line in obj] for obj in objs]
    objs = [[_tofloat(line) for line in obj] for obj in objs]
    objs = [tuple(obj) for obj in objs]
    objs.sort()

    lst = []
    for obj in objs:
        for field in obj[:-1]:
            lst.append('%s,' % (field, ))
        lst.append('%s;\n' % (obj[-1], ))

    return '\n'.join(lst)
180178
def _peek(tokens, n=0):
    """"""
    return tokens.peek(n=n, skip=_is_comment, drop=True)
653220
def parse_stream_raw(self, stream, debug=False):
        """"""
        tokens = tokenize.generate_tokens(stream.readline)
        return self.parse_tokens(tokens, debug)
136295
def getTamilWords( tweet ):
        "
        tweet = TamilTweetParser.cleanupPunct( tweet );
        nonETwords = filter( lambda x: len(x) > 0 , re.split(r'\s+',tweet) );#|"+|\'+|#+
        tamilWords = filter( TamilTweetParser.isTamilPredicate, nonETwords );
        return tamilWords
998302
def get_tokens_unprocessed(self, text, stack=('root',)):
        """"""
        self.content_type = None
        return RegexLexer.get_tokens_unprocessed(self, text, stack)
720506
public function parseUrls($tweet)
    {
        foreach ($this->entities->urls as $url) {
            $tweet = $this->replaceEntityInTweet(
                $tweet,
                $url->url,
                '<a href="' . $url->url . '">' . $url->url . '</a>'
            );
        }

        return $tweet;
    }
229088
def urlize_tweet(tweet):
    
    text = tweet.get('html', tweet['text'])
    for hash in tweet['entities']['hashtags']:
        text = text.replace('#%s' % hash['text'], TWITTER_HASHTAG_URL % (quote(hash['text'].encode("utf-8")), hash['text']))
    for mention in tweet['entities']['user_mentions']:
        text = text.replace('@%s' % mention['screen_name'], TWITTER_USERNAME_URL % (quote(mention['screen_name']), mention['screen_name']))
    tweet['html'] = text
    return tweet