Transclude a MediaWiki page
768260
function transclude($string){
		echo $GLOBALS['wgParser']->parse('{{'.$string.'}}', $this->getSkin()->getRelevantTitle(), new ParserOptions)->getText();
	}
869248
def wiki_page(name, path = nil, version = nil, exact = true)
      wiki = wiki_new
      path = name if path.nil?
      name = extract_name(name) || wiki.index_page
      path = extract_path(path)
      path = '/' if exact && path.nil?

      OpenStruct.new(:wiki => wiki, :page => wiki.paged(name, path, exact, version),
                     :name => name, :path => path)
    end
487728
def produce_examples(shard_ids, wikis_dir, refs_dir, urls_dir, vocab_path,
                     out_filepaths):
  """"""
  # * Join the Wikipedia articles with their references
  # * Run Tf-idf to sort reference paragraphs
  # * Encode the Wikipedia and reference text with the vocabulary
  # * Write out TFRecords of tensorflow.Example
  tf.logging.info("Processing %d input shards into %d output files.",
                  len(shard_ids), len(out_filepaths))

  vocab = text_encoder.SubwordTextEncoder(vocab_path)
  eot_ids = vocab.encode(EOT)

  def example_generator():
    """Generate Example dicts."""
    stats = dict(total_original_wikis=0, total_original_refs=0,
                 total_found_refs=0, ref_lengths=[], wiki_original_refs=[],
                 wiki_found_refs=[], wikis_skipped_no_refs=0,
                 wikis_skipped_short_lead=0, num_wikis_written=0)
    ref_files_by_shard = _references_files_by_shard(refs_dir)
    for shard_id in shard_ids:
      tf.logging.info("Processing shard %d", shard_id)
      wiki_urls = _wiki_urls_for_shard(shard_id, urls_dir)
      tf.logging.info("Loaded wiki URLs for shard")
      refs_content = _references_content(ref_files_by_shard[shard_id])
      tf.logging.info("Loaded reference content for shard")
      for i, wiki in enumerate(_wiki_articles(shard_id, wikis_dir)):
        if not i % 1000:
          tf.logging.info("Processing wiki index %d for shard %d", i, shard_id)
        stats["total_original_wikis"] += 1

        # Get reference content
        wiki_ref_content = []
        ref_urls = wiki_urls[wiki.url]["refs"]
        stats["total_original_refs"] += len(ref_urls)
        stats_wiki_original_refs = len(ref_urls)
        stats_wiki_found_refs = 0
        for ref_url in ref_urls:
          ref_content = refs_content.get(ref_url)
          if not ref_content:
            continue
          stats["total_found_refs"] += 1
          stats["ref_lengths"].append(len(ref_content))
          stats_wiki_found_refs += 1
          wiki_ref_content.append(ref_content)

        stats["wiki_original_refs"].append(stats_wiki_original_refs)
        stats["wiki_found_refs"].append(stats_wiki_found_refs)
        if not wiki_ref_content or len(wiki_ref_content) < _MIN_REFS:
          # No/few refs were found
          stats["wikis_skipped_no_refs"] += 1
          continue

        # Rank reference paragraphs with TFIDF
        wiki_title = _normalize_text(wiki.title)
        ranked_paragraphs = rank_reference_paragraphs(wiki_title,
                                                      wiki_ref_content)

        # Construct inputs from Wiki title and references
        inputs = []
        inputs.extend(vocab.encode(wiki_title))
        inputs.extend(eot_ids)
        for paragraph in ranked_paragraphs:
          if len(inputs) >= 1e6:
            break
          paragraph += " "
          inputs.extend(vocab.encode(paragraph))

        # Construct targets from article sections
        targets, section_boundaries = _encode_wiki_sections(
            wiki.sections, vocab)

        # Skip if lead section is too short
        if (not section_boundaries or
            section_boundaries[0] < _MIN_LEADSECTION_TOKENS):
          stats["wikis_skipped_short_lead"] += 1
          continue

        inputs.append(text_encoder.EOS_ID)
        targets.append(text_encoder.EOS_ID)

        stats["num_wikis_written"] += 1
        yield {
            "inputs": inputs,
            "targets": targets,
            "section_boundaries": section_boundaries,
        }

    tf.logging.info("Total: %d, Skipped: %d",
                    stats["num_wikis_written"],
                    stats["total_original_wikis"] - stats["num_wikis_written"])
    tf.logging.info("Total refs: %d, Skipped refs: %d",
                    stats["total_found_refs"],
                    stats["total_original_refs"] - stats["total_found_refs"])
    stats_fname = os.path.join(os.path.split(out_filepaths[0])[0],
                               "stats.%d.json" % shard_ids[0])
    with tf.gfile.Open(stats_fname, "w") as f:
      f.write(json.dumps(stats))

  generator_utils.generate_files(example_generator(), out_filepaths)
494345
def _wiki_articles(shard_id, wikis_dir=None):
  """"""
  if not wikis_dir:
    wikis_dir = WIKI_CONTENT_DIR
  with tf.Graph().as_default():
    dataset = tf.data.TFRecordDataset(
        cc_utils.readahead(
            os.path.join(wikis_dir, WIKI_CONTENT_FILE % shard_id)),
        buffer_size=16 * 1000 * 1000)

    def _parse_example(ex_ser):
      """Parse serialized Example containing Wikipedia article content."""
      features = {
          "url": tf.VarLenFeature(tf.string),
          "title": tf.VarLenFeature(tf.string),
          "section_titles": tf.VarLenFeature(tf.string),
          "section_texts": tf.VarLenFeature(tf.string),
      }
      ex = tf.parse_single_example(ex_ser, features)
      for k in ex.keys():
        ex[k] = ex[k].values
      ex["url"] = ex["url"][0]
      ex["title"] = ex["title"][0]
      return ex

    dataset = dataset.map(_parse_example, num_parallel_calls=32)
    dataset = dataset.prefetch(100)
    record_it = dataset.make_one_shot_iterator().get_next()

    with tf.Session() as sess:
      while True:
        try:
          ex = sess.run(record_it)
        except tf.errors.OutOfRangeError:
          break

        sections = [
            WikipediaSection(title=text_encoder.to_unicode(title),
                             text=text_encoder.to_unicode(text))
            for title, text in zip(ex["section_titles"], ex["section_texts"])
        ]
        yield WikipediaArticle(
            url=text_encoder.to_unicode(ex["url"]),
            title=text_encoder.to_unicode(ex["title"]),
            sections=sections)
129367
def _media(self):
        
        def static_url(url):
            return staticfiles_storage.url('zinnia_wymeditor/%s' % url)

        media = super(EntryAdminWYMEditorMixin, self).media

        media += Media(
            js=(static_url('js/jquery.min.js'),
                static_url('js/wymeditor/jquery.wymeditor.pack.js'),
                static_url('js/wymeditor/plugins/hovertools/'
                           'jquery.wymeditor.hovertools.js'),
                reverse('admin:zinnia_entry_wymeditor')))
        return media
592349
def _pull_content_revision_parent(self):
        

        if self._revision_id is None:
            query_params = {
                "prop": "extracts|revisions",
                "explaintext": "",
                "rvprop": "ids",
            }
            query_params.update(self.__title_query_param())
            request = self.mediawiki.wiki_request(query_params)
            page_info = request["query"]["pages"][self.pageid]
            self._content = page_info["extract"]
            self._revision_id = page_info["revisions"][0]["revid"]
            self._parent_id = page_info["revisions"][0]["parentid"]
        return self._content, self._revision_id, self._parent_id
249176
def to_modify(self, uid):
        '''
        
        '''

        kwd = {
            'pager': '',

        }
        self.render('wiki_page/page_edit.html',
                    postinfo=MWiki.get_by_uid(uid),
                    kwd=kwd,
                    cfg=CMS_CFG,
                    userinfo=self.userinfo)
1100732
def _update_wikidata(self):
        
        claims = self.data['claims']

        for ent in claims:

            plabel = self.data['labels'].get(ent)  # P (property) label
            if plabel:
                plabel = "%s (%s)" % (plabel, ent)

            claim = []
            for item in claims[ent]:
                ilabel = item
                if utils.is_text(item) and re.match(r'^Q\d+$', item):
                    ilabel = self.data['labels'].get(item)  # Q (item) label
                    if ilabel:
                        ilabel = "%s (%s)" % (ilabel, item)

                if len(claims[ent]) == 1:
                    claim = ilabel
                else:
                    claim.append(ilabel)

            if plabel and ilabel:
                self.data['wikidata'][plabel] = claim
1034601
function buildHelpContents() {
		var pageTpl = '<div class="cke_accessibility_legend" role="document" aria-labelledby="' + id + '_arialbl" tabIndex="-1">%1</div>' +
				'<span id="' + id + '_arialbl" class="cke_voice_label">' + lang.contents + ' </span>',
			sectionTpl = '<h1>%1</h1><dl>%2</dl>',
			itemTpl = '<dt>%1</dt><dd>%2</dd>';

		var pageHtml = [],
			sections = lang.legend,
			sectionLength = sections.length;

		for ( var i = 0; i < sectionLength; i++ ) {
			var section = sections[ i ],
				sectionHtml = [],
				items = section.items,
				itemsLength = items.length;

			for ( var j = 0; j < itemsLength; j++ ) {
				var item = items[ j ],
					itemLegend = item.legend.replace( variablesPattern, replaceVariables );

				// (#9765) If some commands haven't been replaced in the legend,
				// most likely their keystrokes are unavailable and we shouldn't include
				// them in our help list.
				if ( itemLegend.match( variablesPattern ) )
					continue;

				sectionHtml.push( itemTpl.replace( '%1', item.name ).replace( '%2', itemLegend ) );
			}

			pageHtml.push( sectionTpl.replace( '%1', section.name ).replace( '%2', sectionHtml.join( '' ) ) );
		}

		return pageTpl.replace( '%1', pageHtml.join( '' ) );
	}
716712
def parse(self, title, pageid=None):
        
        qry = self.PARSE.substitute(
            WIKI=self.uri,
            ENDPOINT=self.endpoint,
            PAGE=safequote(title) or pageid)

        if pageid and not title:
            qry = qry.replace('&page=', '&pageid=').replace('&redirects', '')

        if self.variant:
            qry += '&variant=' + self.variant

        self.set_status('parse', pageid or title)

        return qry
848132
def html(self):
        
        if self._html is False:
            self._html = None
            query_params = {
                "prop": "revisions",
                "rvprop": "content",
                "rvlimit": 1,
                "rvparse": "",
                "titles": self.title,
            }
            request = self.mediawiki.wiki_request(query_params)
            page = request["query"]["pages"][self.pageid]
            self._html = page["revisions"][0]["*"]
        return self._html
1058975
public void processDocument(WikiDoc doc) {

        String rawArticleName = doc.name;        
        String articleName = StringUtils.unescapeHTML(rawArticleName);
        articleName = articleName.trim().toLowerCase();

        // skip articles that are not text-based or are
        // wikipedia-specific
        if (!isArticleLink(articleName)) {
            LOGGER.fine("skipping non-article document: " + articleName);
            return;
        } 
        // Skip processing articles that exist solely to direct the user to a
        // different article
        else if (articleName.indexOf("#REDIRECT") >= 0 ||
                   doc.text.indexOf("#REDIRECT") >= 0) {
            LOGGER.fine("skipping redirect: " + articleName);
            return;
        }
        LOGGER.log(Level.FINE, "Procesing article {0} with {1} characters",
                   new Object[] { articleName, doc.text.length() });
    
        // Gets the raw tokens contained in the article XML
        StringBuilder rawArticleText = doc.text;

        // Remove all the header and footer information from the article
        LOGGER.finer("extracting raw article text");        
        extractArticle(rawArticleText);

        // Remove any tables
        LOGGER.finer("removing tables");
        removeTables(rawArticleText);       

        // Remove the {{ text }} content from the document first as it sometimes
        // contains [[ text ]], which would be needlessly processed prior to
        // removal if this were called later.
        LOGGER.finer("removing {{text}} from article");
        removeDoubleBraceMarkup(rawArticleText);

        // Next remove the [[ link ]] markup
        LOGGER.finer("removing [[wiki-link]] from article");
        removeWikiLinkMarkup(rawArticleText, articleName);

        // Once the wiki-links are removed, go after external links, which are
        // only a single [link].  Occassinally, these will have text.
        LOGGER.finer("removing [external-link] from article");
        removeExternalLinkMarkup(rawArticleText);
                
        // Once all of the wiki markup has been removed, replace all of the
        // encoded HTML with its equivalents
        LOGGER.finer("unescaping HTML");
        StringUtils.unescapeHTML(rawArticleText);

        // Remove auto-generated Wikipedia warnings which appear in HTML comment
        // text
        LOGGER.finer("removing HTML comments");
        removeHtmlComments(rawArticleText);
        
        String article = rawArticleText.toString();

        // Being removing any tokens according to the options
        if (options.contains(CleanerOption.USE_PREPROCESSOR)) {
            LOGGER.finer("applying preprocessor");
            article = new DocumentPreprocessor().process(article);
        }
        if (options.contains(CleanerOption.FILTER_TOKENS)) {
            LOGGER.finer("filtering tokens");
            article = filterTokens(article);
        }
            
        // Count how many tokens remain in the document after all of the
        // processing stages.  If too few remain, do not write the document
        int finalTokenCount = getTokenCount(article);
        if (finalTokenCount < minTokensPerArticle) {
            LOGGER.log(Level.FINE, "Document {0} contained only {1} tokens"
                       + " and was not printed", new Object[] {
                           articleName, finalTokenCount });
            return;
        }

        if (options.contains(CleanerOption.INCLUDE_TITLES)) {
            processedArticleWriter.print(articleName);
            processedArticleWriter.print(" ");
        }
        
        // Write the remaining part of the article
        processedArticleWriter.println(article);
        processedArticleWriter.flush();
    }
513781
def _continued_query(self, query_params, key="pages"):
        
        query_params.update(self.__title_query_param())

        last_cont = dict()
        prop = query_params.get("prop")

        while True:
            params = query_params.copy()
            params.update(last_cont)

            request = self.mediawiki.wiki_request(params)

            if "query" not in request:
                break

            pages = request["query"][key]
            if "generator" in query_params:
                for datum in pages.values():
                    yield datum
            elif isinstance(pages, list):
                for datum in list(enumerate(pages)):
                    yield datum[1]
            else:
                for datum in pages[self.pageid].get(prop, list()):
                    yield datum

            if "continue" not in request or request["continue"] == last_cont:
                break

            last_cont = request["continue"]
837159
function extractMediaQuery(props){
    var mediaValue = props.length && props[0];
    if(Object.keys(fucss.media).indexOf(mediaValue) !== -1){
      return props.shift();
    }
  }
1243045
function encodePage(page, summary) {
    var file = page.getFile();
    var attributes = page.getAttributes();
    var article = summary.getByPath(file.getPath());

    var result = attributes.toJS();

    if (article) {
        result.title = article.getTitle();
        result.level = article.getLevel();
        result.depth = article.getDepth();

        var nextArticle = summary.getNextArticle(article);
        if (nextArticle) {
            result.next = encodeSummaryArticle(nextArticle);
        }

        var prevArticle = summary.getPrevArticle(article);
        if (prevArticle) {
            result.previous = encodeSummaryArticle(prevArticle);
        }
    }

    result.content = page.getContent();
    result.dir = page.getDir();

    return result;
}
1144546
def links(self):
        """"""
        # <https://example.com/?page=2>; rel="next", <https://example.com/?page=34>; rel="last"'
        # becomes
        # {
        #     'last': {'rel': 'last', 'url': 'https://example.com/?page=34'},
        #     'next': {'rel': 'next', 'url': 'https://example.com/?page=2'},
        # },
        result = {}
        if 'Link' in self.headers:
            value = self.headers['Link']

            for part in re.split(r', *<', value):
                link = {}
                vs = part.split(';')

                # First section is always an url.
                link['url'] = vs.pop(0).strip('\'" <>')

                for v in vs:
                    if '=' in v:
                        key, v = v.split('=')
                        link[key.strip('\'" ')] = v.strip('\'" ')

                rkey = link.get('rel') or link['url']
                result[rkey] = link

        return result
1153134
def add(self, media, filepath, overwrite=True):
        
        with open(filepath, 'rb') as fhandler:
            self._dokuwiki.send('wiki.putAttachment', media,
                                Binary(fhandler.read()), ow=overwrite)