extract the parsely-page meta content from a page
775495
def _get_parselypage(self, body):
        """"""
        parser = ParselyPageParser()
        ret = None
        try:
            parser.feed(body)
        except HTMLParseError:
            pass  # ignore and hope we got ppage
        if parser.ppage is None:
            return

        ret = parser.ppage
        if ret:
            ret = {parser.original_unescape(k): parser.original_unescape(v)
                   for k, v in iteritems(ret)}
        return ret
1008003
def _readMetaAndContent(self,fid):
        
        found_meta_separator=False

        d={}
        # Read lines until we find the meta separator
        while found_meta_separator is False:
            line = fid.readline()
            # Did we find the --- separator it?
            if line[0:len(self.META_SEPARATOR)]==self.META_SEPARATOR:
                found_meta_separator=True
            else:
                key=line.split(':')[0].strip().lower()
                if key in self.WP_META_KEYS:
                    d[key]=line.split(':')[1].strip()
                else:
                    logger.error("wp: Token '%s' not in list of known tokens %s"\
                            %(key,self.WP_META_KEYS))
                    return None

        if not d.has_key('title'):
            print("wp: A title: keyword is required!")

        d['content']=fid.readlines()
        d['content']=''.join(d['content'])

        
        # Let's transfer over to a wordpress post class
        post = WordPressPost()
        post.title=d['title']
        post.content=d['content']
        post.post_status='publish'

        post.terms_names={}
        if d.has_key('tags'):
            post.terms_names['post_tag']=d['tags'].split(',')
        if d.has_key('category'):
            post.terms_names['category']=d['category'].split(',')

        return post
1020143
def rel_links(cls, page):
    """"""
    for match in cls.REL_RE.finditer(page):
      href, rel = match.group(0), match.group(1)
      if rel not in cls.REL_TYPES:
        continue
      href_match = cls.HREF_RE.search(href)
      if href_match:
        href = cls.href_match_to_url(href_match)
        parsed_href = urlparse(href)
        if any(parsed_href.path.endswith(ext) for ext in cls.REL_SKIP_EXTENSIONS):
          continue
        yield href
687796
def _parse_cached(html_dump):
    
    soup = BeautifulSoup(html_dump, "html.parser")
    translations = []
    for trans in soup.find_all("div", class_="translation"):
        word = tuple(t.get_text() for t in trans.select("div.word > h2"))
        trans_list = []
        for part in trans.find_all("div", class_="part-of-speech"):
            pn = part.find("p", class_="part-name")
            if pn:
                pn = pn.get_text().strip("[]")
            meanings = []
            for meaning in part.find_all("div", class_="meaning"):
                m = [mn.get_text() for mn in meaning.select("li > span")]
                examples = []
                for e in meaning.find_all("p"):
                    examples.append([ex.get_text() for ex in e.find_all("span")])
                meanings.append(Meaning(m, examples))
            trans_list.append(PartOfSpeech(pn, meanings))
        translations.append(Translation(word, trans_list))
    return translations
856581
def decode_page(page):
    
    if hasattr(page, 'read'):  # urllib
        if py3k:
            encoding = page.headers.get_param("charset") or "utf-8"
        else:
            encoding = page.headers.getparam("charset") or "utf-8"
        return text_type(page.read(), encoding=encoding)
    else:  # requests?
        encoding = page.headers.get("charset") or "utf-8"
        return text_type(page.content, encoding=encoding)
106007
public function getPageMetas()
    {
        if (!$this->parser) {
            return;
        }

        $nodes = $this->parser->find('meta');

        $metas = array();

        foreach ($nodes as $node) {
            if ($node->hasAttribute('name')) {
                $metas[ $node->getAttribute('name') ] = $node->getAttribute('content');
            } elseif ($node->hasAttribute('property')) {
                $metas[ $node->getAttribute('property') ] = $node->getAttribute('content');
            }
        }

        return $metas;
    }
750493
function getPageMeta() {
        var data = detectFeatures();

        data.title = document.title;
        data.url = window.location.href;
        data.path = document.location.pathname;
        data.referrer = document.referrer;
        data.url_section = window.location.pathname.split('/').filter(function(part){
            return part.length !== 0;
        }).map(function(part){
            return part.replace(/\.[^/.]+$/, "");
        });
        data.hash = window.location.hash;
        data.query_string = window.location.search;

        return data;
    }
665840
def get_metas(cookie, tokens, filelist, dlink=True):
    '''
    '''
    if isinstance(filelist, str):
        filelist = [filelist, ]
    url = ''.join([
        const.PAN_API_URL,
        'filemetas?channel=chunlei&clienttype=0&web=1',
        '&bdstoken=', tokens['bdstoken'],
    ])
    if dlink:
        data = ('dlink=1&target=' +
                encoder.encode_uri_component(json.dumps(filelist)))
    else:
        data = ('dlink=0&target=' +
                encoder.encode_uri_component(json.dumps(filelist)))
    req = net.urlopen(url, headers={
        'Cookie': cookie.sub_output('BDUSS'),
        'Content-type': const.CONTENT_FORM,
        }, data=data.encode())
    if req:
        content = req.data
        return json.loads(content.decode())
    else:
        return None
321507
def parse_meta(meta):
    
    resources = {}
    for name in meta:
        if name.startswith("$"):
            continue
        resources[name] = resource = {}
        for action in meta[name]:
            if action.startswith("$"):
                continue
            url, httpmethod = res_to_url(name, action)
            resource[action] = {
                "url": url,
                "method": httpmethod
            }
    url_prefix = meta.get("$url_prefix", "").rstrip("/")
    return url_prefix, meta["$auth"]["header"].lower(), resources
431339
def _url_to(self, page):
        
        anchor = ""
        if "#" in page:
            page, anchor = page.split("#")
            anchor = "#" + anchor
        meta = self._get_page_meta(page)
        return meta.get("url")
1110052
def _get_page_content(self, page):
        
        src_file = os.path.join(self.pages_dir, page)
        with open(src_file) as f:
            _meta, content = frontmatter.parse(f.read())
            return content
231857
def get_contents(self, el, no_iframe=False):
        """"""
        if not no_iframe or not self.is_iframe(el):
            for content in el.contents:
                yield content
471359
function ReadMetaTags($html)
	{
		// changes anykey=anyvalue to anykey="anyvalue" (only do this when this happens inside tags)
		$regexp = '/ (\\w+?)=([^\\s>"]+)/si';
		$html = preg_replace($regexp, " \$1=\"\$2\"", $html);
		if (preg_match('/<title>(.*?)<\/title>/si', $html, $m)) {
			$this->SetTitle($m[1]);
		}
		preg_match_all('/<meta [^>]*?(name|content)="([^>]*?)" [^>]*?(name|content)="([^>]*?)".*?>/si', $html, $aux);
		$firstattr = $aux[1];
		$secondattr = $aux[3];
		for ($i = 0; $i < count($aux[0]); $i++) {
			$name = ( strtoupper($firstattr[$i]) == "NAME" ) ? strtoupper($aux[2][$i]) : strtoupper($aux[4][$i]);
			$content = ( strtoupper($firstattr[$i]) == "CONTENT" ) ? $aux[2][$i] : $aux[4][$i];
			switch ($name) {
				case "KEYWORDS":
					$this->SetKeywords($content);
					break;
				case "AUTHOR":
					$this->SetAuthor($content);
					break;
				case "DESCRIPTION":
					$this->SetSubject($content);
					break;
			}
		}
	}
131084
function processMeta (markdownContent) {
  let meta = {};
  let metaArr;
  let metaString;
  let metas;

  let yamlObject;

  switch (true) {
    case _metaRegex.test(markdownContent):
      metaArr = markdownContent.match(_metaRegex);
      metaString = metaArr ? metaArr[1].trim() : '';

      if (metaString) {
        metas = metaString.match(/(.*): (.*)/ig);
        metas.forEach(item => {
          const parts = item.split(': ');
          if (parts[0] && parts[1]) {
            meta[cleanString(parts[0], true)] = parts[1].trim();
          }
        });
      }
      break;

    case _metaRegexYaml.test(markdownContent):
      metaArr = markdownContent.match(_metaRegexYaml);
      metaString = metaArr ? metaArr[1].trim() : '';
      yamlObject = yaml.safeLoad(metaString);
      meta = cleanObjectStrings(yamlObject);
      break;

    default:
    // No meta information
  }

  return meta;
}
695551
function normalizePage(content) {
    var data = {
        "head-start": [],
        "head": [],
        "head-end": [],
        body: []
    };

    if (typeof content === "string") {
        data.body.push(content);
    } else {
        ["head-start", "head", "head-end", "body"].forEach(function (section) {
            var sectionContent = content[section];
            if (!sectionContent) {
                return;
            }

            if (!_.isArray(sectionContent)) {
                data[section].push(sectionContent);
            } else {
                data[section] = sectionContent;
            }
        });
    }

    return data;
}
920910
function parse(meta) {
  if (typeof meta !== 'string') {
    throw new Error('`Parse`\'s first argument should be a string')
  }

  return meta.split(/[\r\n]/)
    .filter(function (line) { // remove blank line
      return /\S+/.test(line) &&
        line.indexOf('==UserScript==') === -1 &&
        line.indexOf('==/UserScript==') === -1
    })
    .reduce(function (obj, line) {
      var arr = line.trim().replace(/^\/\//, '').trim().split(/\s+/)
      var key = arr[0].slice(1)
      var value = arr.slice(1).join(' ')

      if (isUndefined(obj[key])) {
        obj[key] = value
      } else if (Array.isArray(obj[key])) {
        obj[key].push(value)
      } else {
        obj[key] = [obj[key], value]
      }

      return obj
    }, {})
}
330003
def parse(page_to_parse):
    """"""
    global _parsed
    if not isinstance(page_to_parse, page.Page):
        raise TypeError("parser.parse requires a parker.Page object.")

    if page_to_parse.content is None:
        raise ValueError("parser.parse requires a fetched parker.Page object.")

    try:
        parsed = _parsed[page_to_parse]
    except KeyError:
        parsed = parsedpage.ParsedPage(
            page=page_to_parse,
            parsed=PyQuery(page_to_parse.content, parser='html')
        )
        _parsed[page_to_parse] = parsed

    return parsed