1. Fetch URL
        2. Run automation.
        3. Return HTML.
        4. Close the tab.
1346
def fetch(self, conf):
        
        url = conf['url']
        # If Firefox is broken, it will raise here, causing kibitzr restart:
        self.driver.set_window_size(1366, 800)
        self.driver.implicitly_wait(2)
        self.driver.get(url)
        try:
            self._run_automation(conf)
            html = self._get_html()
        except:
            logger.exception(
                "Exception occurred while fetching"
            )
            return False, traceback.format_exc()
        finally:
            self._close_tab()
        return True, html
558267
async function main({ isFragment, needsHelp, showVersion, useTabs }) {
  /* eslint-disable no-console */
  const stdin = await getStdin()

  if (showVersion) {
    return console.log(version)
  }

  if (needsHelp || !stdin) {
    return console.log(help)
  }

  const pug = html2pug(stdin, { isFragment, useTabs })
  return console.log(pug)

  /* eslint-enable no-console */
}
19353
def handle_get(self):
        

        response = self.generate_html_documentation().encode('utf-8')

        print('Content-Type: text/html')
        print('Content-Length: %d' % len(response))
        print()
        sys.stdout.flush()
        sys.stdout.buffer.write(response)
        sys.stdout.buffer.flush()
361232
function main (url, configs) {
  const retryOptions = {
    retries: configs.retries
  }

  const fetchOptions = {
    headers,
    timeout: configs.timeout,
    agent: proxy(configs.proxy)
  }

  return pRetry(() => {
    return fetch(url, fetchOptions).then(res => res.text())
  }, retryOptions)
}
411300
function download(url) {
        var a = document.createElement('a');
        a.href = url;
        a.setAttribute('download', 'doc');
        document.body.appendChild(a);
        a.click();
        document.body.removeChild(a);
    }
13735
async def async_get(self, url):
        
        self.reset()
        self.next_link = url
        return await self.async_advance_page()
125369
def Main(url):
    '''
    
    '''
    # The object of Web-Scraping.
    web_scrape = WebScraping()
    # Execute Web-Scraping.
    document = web_scrape.scrape(url)
    # The object of automatic summarization with N-gram.
    auto_abstractor = NgramAutoAbstractor()
    # n-gram object
    auto_abstractor.n_gram = Ngram()
    # n of n-gram
    auto_abstractor.n = 3
    # Set tokenizer. This is japanese tokenizer with MeCab.
    auto_abstractor.tokenizable_doc = MeCabTokenizer()
    # Object of abstracting and filtering document.
    abstractable_doc = TopNRankAbstractor()
    # Execute summarization.
    result_dict = auto_abstractor.summarize(document, abstractable_doc)

    # Output 3 summarized sentences.
    limit = 3
    i = 1
    for sentence in result_dict["summarize_result"]:
        print(sentence)
        if i >= limit:
            break
        i += 1
951614
function scrapeSearchPage(pageUrl) {

    return co(function *() {
        //Load the HTML page
        const $ = cheerio.load(yield request(pageUrl));

        //For each card, scrape some data
        const cards = Array.from($("#cardResults li")).map(el => {
            const $card = $(el);

            //Just scrape the database URL and the image for now
            const url = Url.resolve(SCRAPE_URL, $card.find("a").attr("href"));
            return {
                url: url,
                image: Url.resolve(SCRAPE_URL, $card.find("img").attr("src")),
                id: cardIdFromUrl(url)
            };
        });

        //Work out how many pages in total there are
        const totalText = $('#cards-load-more>div>span').text();
        const numPages = parseInt(/\d of (\d+)/.exec(totalText)[1]);

        return {
            numPages: numPages,
            cards: cards
        };

    })();
}
475081
function downloadFromClick(page, ph, selector, callback) {
    const HREF_KEY = 'href';
    async.waterfall([
        function (callback) {
            getSelectorAttribute(page, ph, [selector, HREF_KEY], callback);
        },
        function (page, ph, relativeHref, callback) {
            getCurrentUrl(page, ph, function (error, page, ph, currentUrl) {
                if (error) {
                    callback(error, page, ph);
                    return;
                }
                callback(null, page, ph, currentUrl + relativeHref);
            });
        },
        function (page, ph, url, callback) {
            downloadFromUrl(page, ph, url, callback);
        }
    ], callback);
}
1270142
function(url) {
		return new Promise(function (resolve) {
			self.roptions.url = url;
			request(self.roptions, function (err, response, body) {
				if (!err && response.statusCode === 200) {
					var $ = cheerio.load(body);

					// no results
					if (!$('.result').length) {
						return resolve(result);
					}
					
					$('.result').each(function() {
						var href = $(this).find('a').attr('href');
						if(href.toString().indexOf("http") !== 0) {
							result.push(base + href); 							
						} else {
							result.push(href); 							
						}
					});

					if (num && result.length > num) {
						return resolve(result.slice(0,num));
					}

					// search for current page and select next one
					var page = $('#nextPage').attr('href');
					if (page) {
						resolve(nextPage(base + page));
					}
					else {
						resolve(result);
					}
				} else {
					resolve(result)
				}
			});
		});
	}
405254
def close_tab(self, *args):
        
        prompt_cfg = self.settings.general.get_int('prompt-on-close-tab')
        self.get_notebook().delete_page_current(prompt=prompt_cfg)
314434
def do_GET(self):
    """"""
    if not self._client_allowed():
      return

    try:
      (_, _, path, query, _) = urlsplit(self.path)
      params = parse_qs(query)
      # Give each handler a chance to respond.
      for prefix, handler in self._GET_handlers:
        if self._maybe_handle(prefix, handler, path, params):
          return
      # If no path specified, default to showing the list of all runs.
      if path == '/':
        self._handle_runs('', {})
        return

      content = 'Invalid GET request {}'.format(self.path).encode('utf-8'),
      self._send_content(content, 'text/html', code=400)
    except (IOError, ValueError):
      pass
40852
function analyzeHTML(result, $, callback) {

  // if $ is note defined, this is not a HTML page with an http status 200
  if (! $) {
    return callback();
  }

  log.debug({"url" : result.url, "step" : "analyzeHTML", "message" : "Start check HTML code"});

  async.parallel([

    async.apply(crawlHrefs, result, $),
    async.apply(crawlLinks, result, $),
    async.apply(crawlScripts, result, $),
    async.apply(crawlImages, result, $),

  ], callback);

}
1144687
def fetch(**kwargs):
    '''
    
    '''
    # fetch continues when no controlling terminal is present
    pre = ''
    post = ''
    run_args = {}
    if float(__grains__['osrelease']) >= 10.2:
        post += '--not-running-from-cron'
    else:
        pre += ' env PAGER=cat'
        run_args['python_shell'] = True
    return _wrapper('fetch', pre=pre, post=post, run_args=run_args, **kwargs)
556311
async function _process(
  args,
  { fetch },
  {
    _omitScript,
    maxWidth,
    hideMedia,
    hideThread,
    align,
    lang,
    theme,
    linkColor,
    widgetType
  }
) {
  const params = {
    url: args[0],
    omitScript: _omitScript,
    maxWidth,
    hideMedia,
    hideThread,
    align,
    lang,
    theme,
    linkColor,
    widgetType
  }
  try {
    const apiUrl = `https://api.twitter.com/1/statuses/oembed.json?${getQuery(
      params
    )}`
    const res = await (isBrowser ? jsonp : fetch)(apiUrl)
    return await res.json()
  } catch (e) {
    return {
      html: ""
    }
  }
}
163795
function getTabsWithURL(aUrl) {
  var tabs = [ ];

  var uri = utils.createURI(aUrl, null, null);

  var wm = Cc["@mozilla.org/appshell/window-mediator;1"].
           getService(Ci.nsIWindowMediator);
  var winEnum = wm.getEnumerator("navigator:browser");

  // Iterate through all windows
  while (winEnum.hasMoreElements()) {
    var window = winEnum.getNext();
 
    // Don't check windows which are about to close or don't have gBrowser set
    if (window.closed || !("gBrowser" in window))
      continue;

    // Iterate through all tabs in the current window
    var browsers = window.gBrowser.browsers;
    for (var i = 0; i < browsers.length; i++) {
      var browser = browsers[i];
      if (browser.currentURI.equals(uri)) {
        tabs.push({
          controller : new mozmill.controller.MozMillController(window),
          index : i
        });
      }
    }
  }

  return tabs;
}
750617
def Get_Page(self):
		'''
			
		'''

		# Simply encode the correct url as a string
		url = 'https://www.aramex.com/express/track-results-multiple.aspx?ShipmentNumber='
		url += self.tracking_no

		driver = webdriver.PhantomJS()			# create a selenium webdriver
		driver.get(url)							# make it send a request with the above url
		self.wait_till_page_load(driver,10)		# wait till the page is fully loaded
		self.page = driver.page_source		# store the html source
		driver.quit()