Construct a Post from a feedparser entry and save/update it in db
160330
def process_entry(self, entry):
		''

		from feedjack.models import Post, Tag

		## Construct a Post object from feedparser entry (FeedParserDict)
		post = Post(feed=self.feed)
		post.link = entry.get('link', self.feed.link)
		post.title = entry.get('title', post.link)
		post.guid = self._get_guid(entry)

		if 'author_detail' in entry:
			post.author = entry.author_detail.get('name', '')
			post.author_email = entry.author_detail.get('email', '')
		if not post.author: post.author = entry.get('author', entry.get('creator', ''))
		if not post.author_email: post.author_email = 'nospam@nospam.com'

		try: post.content = entry.content[0].value
		except: post.content = entry.get('summary', entry.get('description', ''))

		# Try to get the post date from "updated" then "published" then "created"
		ts_parsed = ts_raw = None
		for k in self.post_timestamp_keys:
			try:
				post.date_modified = get_modified_date(
					entry.get('{0}_parsed'.format(k)), entry.get(k) )
			except ValueError as err:
				log.warn( 'Failed to process post timestamp:'
					' {0} (feed_id: {1}, post_guid: {2})'.format(err, self.feed.id, post.guid) )
			if post.date_modified: break

		post.comments = entry.get('comments', '')

		enclosures = entry.get('enclosures', list())
		if 'media_content' in entry:
			for mc in entry.media_content:
				if 'url' in mc: e = dict(href=mc['url'], medium=mc.get('medium', 'image'))
				else: e = entry.media_content
				e['type'] = 'application/x-media-content' # special ct for these things
				enclosures.append(e)
			assert enclosures, enclosures
		post.enclosures = enclosures

		## Get a list of tag objects from an entry
		# Note that these objects can't go into m2m field until properly saved
		fcat = list()
		if entry.has_key('tags'):
			for tcat in entry.tags:
				qcat = tcat.label if tcat.label is not None else tcat.term
				if not qcat: continue

				qcat = qcat.strip()
				if ',' in qcat or '/' in qcat: qcat = qcat.replace(',', '/').split('/')
				else: qcat = [qcat]

				for zcat in qcat:
					tagname = ' '.join(zcat.lower().split()).strip()[:255]
					if not tagname: continue
					if not Tag.objects.filter(name=tagname):
						cobj = Tag(name=tagname)
						cobj.save()
					fcat.append(Tag.objects.get(name=tagname))

		## Some feedback
		post_base_fields = 'title link guid author author_email'.split()

		log.debug('[{0}] Entry\n{1}'.format(self.feed.id, '\n'.join(
			['  {0}: {1}'.format(key, getattr(post, key)) for key in post_base_fields]
			+ ['tags: {0}'.format(' '.join(it.imap(op.attrgetter('name'), fcat)))] )))

		## Store / update a post
		if post.guid in self.postdict: # post exists, update if it was modified (and feed is mutable)
			post_old = self.postdict[post.guid]
			changed = post_old.content != post.content or (
				post.date_modified and post_old.date_modified != post.date_modified )

			if not self.feed.immutable and changed:
				retval = ENTRY_UPDATED
				log.extra('[{0}] Updating existing post: {1}'.format(self.feed.id, post.link))
				# Update fields
				for field in post_base_fields + ['content', 'comments']:
					setattr(post_old, field, getattr(post, field))
				post_old.date_modified = post.date_modified or post_old.date_modified
				# Update tags
				post_old.tags.clear()
				for tcat in fcat: post_old.tags.add(tcat)
				post_old.save()
			else:
				retval = ENTRY_SAME
				log.extra( ( '[{0}] Post has not changed: {1}' if not changed else
					'[{0}] Post changed, but feed is marked as immutable: {1}' )\
						.format(self.feed.id, post.link) )

		else: # new post, store it into database
			retval = ENTRY_NEW
			log.extra( '[{0}] Saving new post: {1} (timestamp: {2})'\
				.format(self.feed.id, post.guid, post.date_modified) )

			# Try hard to set date_modified: feed.modified, http.modified and now() as a last resort
			if not post.date_modified and self.fpf:
				try:
					post.date_modified = get_modified_date(
						self.fpf.feed.get('modified_parsed') or self.fpf.get('modified_parsed'),
						self.fpf.feed.get('modified') or self.fpf.get('modified') )
				except ValueError as err:
					log.warn(( 'Failed to process feed/http timestamp: {0} (feed_id: {1},'
						' post_guid: {2}), falling back to "now"' ).format(err, self.feed.id, post.guid))
				if not post.date_modified:
					post.date_modified = timezone.now()
					log.debug(( '[{0}] Using current time for post'
						' ({1}) timestamp' ).format(self.feed.id, post.guid))
				else:
					log.debug(
						'[{0}] Using timestamp from feed/http for post ({1}): {2}'\
						.format(self.feed.id, post.guid, post.date_modified) )

			if self.options.hidden: post.hidden = True
			try: post.save()
			except IntegrityError:
				log.error( 'IntegrityError while saving (supposedly) new'\
					' post with guid: {0.guid}, link: {0.link}, title: {0.title}'.format(post) )
				raise
			for tcat in fcat: post.tags.add(tcat)
			self.postdict[post.guid] = post

		return retval
811649
def parse_feed(content):
        
        feed = feedparser.parse(content)
        articles = []
        for entry in feed['entries']:
            article = {
                'title': entry['title'],
                'link': entry['link']
            }
            try:
                article['media'] = entry['media_content'][0]['url']
            except KeyError:
                article['media'] = None
            articles.append(article)
        return articles
198100
def post_parse_action(self, entry):
        """"""
        if 'source_host' in entry.keys():
            host = self.ip_port_regex.findall(entry['source_host'])
            if host:
                hlist = host[0].split('.')
                entry['source_host'] = '.'.join(hlist[:4])
                entry['source_port'] = hlist[-1]
        if 'dest_host' in entry.keys():
            host = self.ip_port_regex.findall(entry['dest_host'])
            if host:
                hlist = host[0].split('.')
                entry['dest_host'] = '.'.join(hlist[:4])
                entry['dest_port'] = hlist[-1]

        return entry
977414
function postSetTags(post_id, tags, cb) {
  var ids = _.pluck(tags, 'id')
    , q1 = 'DELETE FROM posts_tags WHERE post_id = $1'
    , q2 = 'INSERT INTO posts_tags (post_id, tag_id) VALUES ' +
    ids.map(function(id) {
      return '(' + post_id + ',' + id + ')';
    }).join(', ');

  db.getClient(function(err, client, done) {
    client.query(q1, [post_id], function(err, r) {
      if(err) { return cb(err); }
      client.query(q2, function(err, r) {
        if(err) {
          cb(err);
          done(err);
        } else {
          cb(null, r.rowCount);
          done();
        }
      });
    });
  });
}
605500
def setup_parse(self, inputstring, document):
        """"""
        self.inputstring = inputstring
        self.document = document
554433
public function feed()
    {
        if (isset($_POST[$this->name])) {
            
            $structure = json_decode($_POST[$this->name], true);
            if ($structure !== false) {
                $description = new tao_helpers_form_data_UploadFileDescription(array_key_exists('name',
                    $structure) ? $structure['name'] : null,
                    array_key_exists('size', $structure) ? $structure['size'] : null,
                    array_key_exists('type', $structure) ? $structure['type'] : null,
                    array_key_exists('uploaded_file', $structure) ? $structure['uploaded_file'] : null,
                    array_key_exists('action', $structure) ? $structure['action'] : null);
                $this->setValue($description);
            } else {
                // else, no file was selected by the end user.
                // set the value as empty in order
                $this->setValue($_POST[$this->name]);
            }
        }
    }
573043
def update(self, status, source=None, params={}):
        ""
        params = params.copy()
        params['status'] = status
        if source:
            params['source'] = source
        return self.__parsed_post(self.__post('/statuses/update.xml', params),
            txml.parseUpdateResponse)
925121
function parseUpdateInfo (node, nodeType, feedType) {
  var updateInfo = {}, temp;

  if (temp = utils.get(node['sy:updatefrequency'])) {
    updateInfo.frequency = temp;
  }
  if (temp = utils.get(node['sy:updateperiod'])) {
    updateInfo.period = temp;
  }
  if (temp = utils.get(node['sy:updatebase'])) {
    updateInfo.base = temp;
  }
  if (temp = utils.get(node.ttl)) {
    updateInfo.ttl = temp;
  }

  return Object.keys(updateInfo).length ? updateInfo : null;
}
1162685
def _store_post(self, stored_entry, entry=None):
        
        #        stored_entry.content_type = utils.get_source_setting(ds.type, 'type')
        if stored_entry.published is None:
            stored_entry.published = self._get_dummy_datetime()
        if stored_entry.updated is None:
            stored_entry.updated = self._get_dummy_datetime()
        #        print 'Post: %s' % stored_entry.title
        #        print 'Attributes: %s' % stored_entry.attributes
        stored_entry.save()
        self.posts.add(stored_entry)
        return None
64702
public function parse()
    {
        if ($this->arfMail !== true) {
            $this->feedName = 'default';

            // As this is a generic FBL parser we need to see which was the source and add the name
            // to the report, so its origin is clearly shown.
            $source = $this->parsedMail->getHeader('from');
            foreach (config("{$this->configBase}.parser.aliases") as $from => $alias) {
                if (preg_match($from, $source)) {
                    // If there is an alias, prefer that name instead of the from address
                    $source = $alias;

                    // If there is an more specific feed configuration prefer that config over the default
                    if (!empty(config("{$this->configBase}.feeds.{$source}"))) {
                        $this->feedName = $source;
                    }
                }
            }

            // If feed is known and enabled, validate data and save report
            if ($this->isKnownFeed() && $this->isEnabledFeed()) {
                // To get some more consitency, remove "\r" from the report.
                $this->arfMail['report'] = str_replace("\r", "", $this->arfMail['report']);

                // Build up the report
                preg_match_all(
                    "/([\w\-]+): (.*)[ ]*\n/m",
                    $this->arfMail['report'],
                    $matches
                );

                $report = array_combine($matches[1], $matches[2]);

                if (empty($report['Received-Date'])) {
                    if (!empty($report['Arrival-Date'])) {
                        $report['Received-Date'] = $report['Arrival-Date'];
                        unset($report['Arrival-Date']);
                    }
                }

                // Now parse the headers from the spam messages and add it to the report
                if (!empty($this->arfMail['evidence'])) {
                    $spamMessage = new MimeParser();
                    $spamMessage->setText($this->arfMail['evidence']);
                    $report['headers'] = $spamMessage->getHeaders();
                } else {
                    $this->failed(
                        'The e-mail received at the parser is not RFC822 compliant, and therefor not a FBL message'
                    );
                }

                // Also add the spam message body to the report
                $report['body'] = $spamMessage->getMessageBody();

                // Sanity check
                if ($this->hasRequiredFields($report) === true) {

                    // incident has all requirements met, filter and add!
                    $report = $this->applyFilters($report);

                    $incident = new Incident();
                    $incident->source      = $source; // FeedName
                    $incident->source_id   = false;
                    $incident->ip          = $report['Source-IP'];
                    $incident->domain      = false;
                    $incident->class       = config("{$this->configBase}.feeds.{$this->feedName}.class");
                    $incident->type        = config("{$this->configBase}.feeds.{$this->feedName}.type");
                    $incident->timestamp   = strtotime($report['Received-Date']);
                    $incident->information = json_encode($report);

                    $this->incidents[] = $incident;

                }
            }
        }

        return $this->success();
    }
729390
def _get_post_data_to_create_dns_entry(self, rtype, name, content, identifier=None):
        
        is_update = identifier is not None
        if is_update:
            records = self._list_records_internal(identifier=identifier)
            assert len(records) == 1, 'ID is not unique or does not exist'
            record = records[0]
            LOGGER.debug('Create post data to update record: %s', record)

        data = {
            'id': str(identifier) if is_update else '',
            'action': 'save',
            'name': name,
            'type': rtype,
            'content': content,
            'prio': str(record['priority']) if is_update else '10',
            'ttl': str(record['ttl']) if is_update else '360',
            'commit': ''
        }
        ttl = self._get_lexicon_option('ttl')
        if ttl and ttl > 360:
            data['ttl'] = str(ttl)

        prio = self._get_lexicon_option('priority')
        if prio and prio > 0:
            data['prio'] = str(prio)

        return data
1074394
private void process(JsonParser jsonParser) throws IOException, ODataUnmarshallingException {
        if (jsonParser.getCurrentToken() == JsonToken.FIELD_NAME) {
            LOG.info("Starting to parse {} token", jsonParser.getCurrentName());
            String key = jsonParser.getCurrentName();
            jsonParser.nextToken();

            JsonToken token = jsonParser.getCurrentToken();
            if (token == JsonToken.START_ARRAY) {
                if (JsonConstants.VALUE.equals(key)) {
                    throw new ODataUnmarshallingException("Feed is not supported");
                }
                values.put(key, getCollectionValue(jsonParser));
            } else if (token == JsonToken.START_OBJECT) {
                values.put(key, getEmbeddedObject(jsonParser));
            } else {
                if (token.equals(JsonToken.VALUE_NULL)) {
                    values.put(key, null);
                } else {
                    values.put(key, jsonParser.getText());
                }
            }
        }
    }
325028
def from_string(self, html_string):
        """"""
        self._html_parser._reset().feed(html_string)
        return self._html_parser.result
819425
def main():
    """"""
    global config
    parser = OptionParser(version='%prog v' + __version__)
    parser.add_option('-c', '--config', default='config.ini',
                      help='Location of config file (default: %default)',
                      metavar='FILE')
    parser.add_option('-a', '--all', action='store_true', default=False,
                      dest='all',
                      help='Send all RSS items as tweet')
    parser.add_option('-l', '--limit', dest='limit',
                      default=10,
                      help='Use with all parameters, send the first 10 feeds as a tweet')
    (options, args) = parser.parse_args()
    config = SafeConfigParser()
    if not config.read(options.config):
        print 'Could not read config file'
        sys.exit(1)
    rss_uri = config.get('rss', 'uri')
    feed = feedparser.parse(rss_uri)
    # lots of scary warnings about possible security risk using this method
    # but for local use I'd rather do this than a try-catch with open()
    if not os.path.isfile('cache.dat'):
        # make a blank cache file
        cPickle.dump({'id': None}, open('cache.dat', 'wb'), -1)

    cache = cPickle.load(open('cache.dat'))
    if options.all:
        tweet_count = 0
        for entry in feed['entries']:
            rss = {
                'id': entry['id'],
                'link': entry['link'],
                'title': entry['title'],
                'summary': entry['summary'],
                'hashtag': ' '.join(['#%s' % i for i in entry['tags'][0]['term'].split()[:2]]),
            }
            post_update('%s %s %s' % (rss['title'], rss['link'], rss['hashtag']))

            # We keep the first feed in the cache, to use rss2twitter in normal mode the next time
            if tweet_count == 0:
                cPickle.dump(rss, open('cache.dat', 'wb'), -1)

            tweet_count += 1
            if tweet_count >= options.limit:
                break
    else:
        rss = {
            'id': feed['entries'][0]['id'],
            'link': feed['entries'][0]['link'],
            'title': feed['entries'][0]['title'],
            'summary': feed['entries'][0]['summary'],
            'hashtag': ' '.join(['#%s' % i for i in feed['entries'][0]['tags'][0]['term'].split()[:2]]),
        }
        # compare with cache
        if cache['id'] != rss['id']:
            #print 'new post'
            post_update('%s %s %s' % (rss['title'], rss['link'], rss['hashtag']))
            cPickle.dump(rss, open('cache.dat', 'wb'), -1)
624873
def feed(self, data):
        
        self.reset_purified()
        HTMLParser.feed(self, data)
        return self.html()
228912
private function createRSS()
    {
        $file = fopen($this->rssFile, "w+");

        $stmt = $this->db->prepare('SELECT * FROM ' . $this->table . ' ORDER BY CREATED DESC LIMIT ?');
        $stmt->execute([$this->newsCount]);
        $res = $stmt->fetchAll();

        $xmlVersion = '<?xml version="1.0" encoding="UTF-8" ?>';

        $startString = <<<EOD
{$xmlVersion}
<rss version="2.0">
<channel>
    <title>{$this->feedDescription['title']}</title>
    <link>{$this->feedDescription['link']}</link>
    <description>{$this->feedDescription['description']}</description>
EOD;
        fwrite($file, $startString);

        foreach ($res as $post) {
            $date = date("D, d M y H:i:s O", strtotime($post->CREATED));
            $string = <<<EOD

    <item>
        <title>{$post->TITLE}</title>
        <link>{$post->LINK}</link>
        <description>{$post->DESCRIPTION}</description>
        <pubDate>{$date}</pubDate>
    </item>
EOD;
            fwrite($file, $string);
        }

        fwrite($file, '
</channel>
</rss>');
    }
1308665
def post(url, data=None, json=None, **kwargs):
    """"""
    _set_content_type(kwargs)
    if _content_type_is_json(kwargs) and data is not None:
        data = dumps(data)
    _log_request('POST', url, kwargs, data)
    response = requests.post(url, data, json, **kwargs)
    _log_response(response)
    return response