TODO(dfreedm): there may be a better factoring here extract absolute urls from the text (full of relative urls)
504180
function(text, base) {
      var matches = [];
      var matched, u;
      while ((matched = this.regex.exec(text))) {
        u = new URL(matched[1], base);
        matches.push({matched: matched[0], url: u.href});
      }
      return matches;
    }
1196573
function getUrlToken({ tokens, intEmphasis, num }) {
  const urlContent = `(${getAttr(tokens[num].attrs, 'href')})`;
  const urlLength = urlContent.length;
  const punctuation1 = {
    type: "punctuation",
    content: "[",
    length: 1
  };
  const punctuation2 = {
    type: "punctuation",
    content: "]",
    length: 1
  };
  const urlContentObj = {
    type: "punctuation",
    content: urlContent,
    length: urlLength
  };
  const content = Array.isArray(intEmphasis) ?
    [punctuation1, ...intEmphasis, punctuation2, urlContentObj] :
    [punctuation1, intEmphasis, punctuation2, urlContentObj];
  return {
    type: "url",
    content,
    length: Array.isArray(intEmphasis) ? getTokensLength(content) : 1 + intEmphasis.length + 1 + urlLength
  };
}
1282170
function doAutoLinks(text) {

        text = text.replace(/<((https?|ftp|dict):[^'">\s]+)>/gi,"<a href=\"$1\">$1</a>");
        text = text.replace(/<(?:mailto:)?([-.\w]+\@[-a-z0-9]+(\.[-a-z0-9]+)*\.[a-z]+)>/gi, function(wholeMatch,m1) {
            return encodeEmailAddress(unescapeSpecialChars(m1));
        });

        return text;
    }
252051
def find_urls(self, text, only_unique=False):
        
        urls = self.gen_urls(text)
        urls = OrderedDict.fromkeys(urls) if only_unique else urls
        return list(urls)
845530
function(url){

	var retlist = [];

	// Pull parameters.
	var tmp = url.split('?');
	var path = '';
	var parms = [];
	if( ! tmp[1] ){ // catch bad url--nothing before '?'
	    parms = tmp[0].split('&');
	}else{ // normal structure
	    path = tmp[0];
	    parms = tmp[1].split('&');
	}

	// Decompose parameters.
	each(parms, function(p){
	    var c = _first_split('=', p);
	    if( ! c[0] && ! c[1] ){
		retlist.push([p]);
	    }else{
		retlist.push(c);		  
	    }
	});
	
	return retlist;
    }
753240
def _ExtractYandexSearchQuery(self, url):
    
    if 'text=' not in url:
      return None
    _, _, line = url.partition('text=')
    before_and, _, _ = line.partition('&')
    if not before_and:
      return None
    yandex_search_url = before_and.split()[0]

    return yandex_search_url.replace('+', ' ')
1123753
def extract_with_context(lst, pred, before_context, after_context):
    
    rval = []

    start = 0
    length = 0
    while start < len(lst):
        usedfirst = False
        usedlast = False
        # Extend to the next match.
        while start + length < len(lst) and length < before_context + 1 \
                and not pred(lst[start + length]):
            length += 1

        # Slide to the next match.
        while start + length < len(lst) and not pred(lst[start + length]):
            start += 1

        # If there was no next match, abort here (it's easier
        # to do this than to try to detect this case later).
        if start + length == len(lst):
            break

        # Now extend repeatedly until we can't find anything.
        while start + length < len(lst) and pred(lst[start + length]):
            extendlength = 1
            # Read in the 'after' context and see if it holds a URL.
            while extendlength < after_context + 1 and start + length + \
                    extendlength < len(lst) and \
                    not pred(lst[start + length + extendlength]):
                extendlength += 1
            length += extendlength
            if start + length < len(lst) and not pred(lst[start + length]):
                # Didn't find a matching line, so we either
                # hit the end or extended to after_context + 1..
                #
                # Now read in possible 'before' context
                # from the next URL; if we don't find one,
                # we discard the readahead.
                extendlength = 1
                while extendlength < before_context and start + length + \
                        extendlength < len(lst) and \
                        not pred(lst[start + length + extendlength]):
                    extendlength += 1
                if start + length + extendlength < len(lst) and \
                        pred(lst[start + length + extendlength]):
                    length += extendlength

        if length > 0 and start + length <= len(lst):
            if start == 0:
                usedfirst = True
            if start + length == len(lst):
                usedlast = True
            rval.append((lst[start:start + length], usedfirst, usedlast))

        start += length
        length = 0
    return rval
1143857
def _extract_links_from_asset_tags_in_text(self, text):
        
        # Extract asset tags from instructions text
        asset_tags_map = self._extract_asset_tags(text)
        ids = list(iterkeys(asset_tags_map))
        if not ids:
            return {}

        # asset tags contain asset names and ids. We need to make another
        # HTTP request to get asset URL.
        asset_urls = self._extract_asset_urls(ids)

        supplement_links = {}

        # Build supplement links, providing nice titles along the way
        for asset in asset_urls:
            title = clean_filename(
                asset_tags_map[asset['id']]['name'],
                self._unrestricted_filenames)
            extension = clean_filename(
                asset_tags_map[asset['id']]['extension'].strip(),
                self._unrestricted_filenames)
            url = asset['url'].strip()
            if extension not in supplement_links:
                supplement_links[extension] = []
            supplement_links[extension].append((url, title))

        return supplement_links
1259727
def url_to_parts(url):
    

    if not url:
        return None

    scheme, netloc, path, query, fragment = _urlsplit(url)

    if not path or path == '/':
        path = []
    else:
        path = path.strip('/').split('/')

    if not query:
        query = {}
    else:
        query = _parse_qs(query)

    return _urllib_parse.SplitResult(scheme, netloc, path, query, fragment)
1213527
function getUrlVars(string) {
  var vars = [],
    hash,
    hashes,
    str = string.toString();

  if (str.match('&#038;')) {
    str = str.replace("&#038;", "&");
  } else if (str.match('&#38;')) {
    str = str.replace("&#38;", "&");
  } else if (str.match('&amp;')) {
    str = str.replace("&amp;", "&");
  }
  if (str.match('#')) {
    str = str.split('#')[0];
  }
  hashes = str.slice(str.indexOf('?') + 1).split('&');

  for(var i = 0; i < hashes.length; i++) {
    hash = hashes[i].split('=');
    vars.push(hash[0]);
    vars[hash[0]] = hash[1];
  }

  return vars;
}
849023
def bibcode_from_url(cls, url):
        
        try:
            code = url.split('/abs/')
            code = code[1].strip()
            return code
        except:
            return None
634973
def extract_from_text(text):
    
    return tools.remove_duplicates([i[0]
                                    for i in REGEX.findall(text) if i != ''])
985390
def extract_original_links(base_url, bs4):
    
    valid_url = convert_invalid_url(base_url)

    url = urlparse(valid_url)

    base_url = '{}://{}'.format(url.scheme, url.netloc)

    base_url_with_www = '{}://www.{}'.format(url.scheme, url.netloc)

    links = extract_links(bs4)

    result_links = [anchor for anchor in links if anchor.startswith(base_url)]

    result_links_www = [anchor for anchor in links if anchor.startswith(base_url_with_www)]

    return list(set(result_links + result_links_www))
1263760
def resolve_relative_rst_links(text: str, base_link: str):
    
    document = parse_rst(text)
    visitor = SimpleRefCounter(document)
    document.walk(visitor)
    for target in visitor.references:
        name = target.attributes['name']
        uri = target.attributes['refuri']
        new_link = '`{} <{}{}>`_'.format(name, base_link, uri)
        if name == uri:
            text = text.replace('`<{}>`_'.format(uri), new_link)
        else:
            text = text.replace('`{} <{}>`_'.format(name, uri), new_link)
    return text
1024856
def safe_uriref(text):
    
    url_ = url.parse(text).sanitize().deuserinfo().canonical()
    return URIRef(url_.punycode().unicode())
84338
private Map<String, String> extractAbbrPairs(Reader r) {
        Map<String, String> candidates = new HashMap<String, String>();
        String str, tmpStr, longForm = "", shortForm = "";
        String currSentence = "";
        int openParenIndex, closeParenIndex = -1, sentenceEnd, newCloseParenIndex, tmpIndex = -1;
        boolean newParagraph = true;
        StringTokenizer shortTokenizer;

        try {

            BufferedReader fin = new BufferedReader(r);
            while ((str = fin.readLine()) != null) {
                if (str.length() == 0 || newParagraph && !Character.isUpperCase(str.charAt(0))) {
                    currSentence = "";
                    newParagraph = true;
                    continue;
                }
                newParagraph = false;
                str += " ";
                currSentence += str;
                openParenIndex = currSentence.indexOf(" (");
                do {
                    if (openParenIndex > -1) {
                        openParenIndex++;
                    }
                    sentenceEnd = Math.max(currSentence.lastIndexOf(". "), currSentence.lastIndexOf(", "));
                    if ((openParenIndex == -1) && (sentenceEnd == -1)) {
                        // Do nothing
                    } else if (openParenIndex == -1) {
                        currSentence = currSentence.substring(sentenceEnd + 2);
                    } else if ((closeParenIndex = currSentence.indexOf(')', openParenIndex)) > -1) {
                        sentenceEnd = Math.max(currSentence.lastIndexOf(". ", openParenIndex), currSentence.lastIndexOf(", ",
                                openParenIndex));
                        if (sentenceEnd == -1) {
                            sentenceEnd = -2;
                        }
                        longForm = currSentence.substring(sentenceEnd + 2, openParenIndex);
                        shortForm = currSentence.substring(openParenIndex + 1, closeParenIndex);
                    }
                    if (shortForm.length() > 0 || longForm.length() > 0) {
                        if (shortForm.length() > 1 && longForm.length() > 1) {
                            if ((shortForm.indexOf('(') > -1)
                                    && ((newCloseParenIndex = currSentence.indexOf(')', closeParenIndex + 1)) > -1)) {
                                shortForm = currSentence.substring(openParenIndex + 1, newCloseParenIndex);
                                closeParenIndex = newCloseParenIndex;
                            }
                            if ((tmpIndex = shortForm.indexOf(", ")) > -1) {
                                shortForm = shortForm.substring(0, tmpIndex);
                            }
                            if ((tmpIndex = shortForm.indexOf("; ")) > -1) {
                                shortForm = shortForm.substring(0, tmpIndex);
                            }
                            shortTokenizer = new StringTokenizer(shortForm);
                            if (shortTokenizer.countTokens() > 2 || shortForm.length() > longForm.length()) {
                                // Long form in ( )
                                tmpIndex = currSentence.lastIndexOf(" ", openParenIndex - 2);
                                tmpStr = currSentence.substring(tmpIndex + 1, openParenIndex - 1);
                                longForm = shortForm;
                                shortForm = tmpStr;
                                if (!hasCapital(shortForm)) {
                                    shortForm = "";
                                }
                            }
                            if (isValidShortForm(shortForm)) {
                                List<String> list = extractAbbrPair(shortForm.trim(), longForm.trim());
                                if (list.size() == 2) {
                                    candidates.put(list.get(0), list.get(1));
                                }
                            }
                        }
                        currSentence = currSentence.substring(closeParenIndex + 1);
                    } else if (openParenIndex > -1) {
                        if ((currSentence.length() - openParenIndex) > 200) {
                            // Matching close paren was not found
                            currSentence = currSentence.substring(openParenIndex + 1);
                        }
                        break; // Read next line
                    }
                    shortForm = "";
                    longForm = "";
                } while ((openParenIndex = currSentence.indexOf(" (")) > -1);
            }
            fin.close();
        } catch (Exception ioe) {
            ioe.printStackTrace();
            System.out.println(currSentence);
            System.out.println(tmpIndex);
        }
        return candidates;
    }
1031852
def _extract_url_and_title(self, text, start):
        """"""
        # text[start] equals the opening parenthesis
        idx = self._whitespace.match(text, start + 1).end()
        if idx == len(text):
            return None, None
        end_idx = idx
        has_anglebrackets = text[idx] == "<"
        if has_anglebrackets:
            end_idx = self._find_balanced(text, end_idx+1, "<", ">")
        end_idx = self._find_balanced(text, end_idx, "(", ")")
        match = self._inline_link_title.search(text, idx, end_idx)
        if not match:
            return None, None
        url = text[idx:match.start()]
        if has_anglebrackets:
            url = self._strip_anglebrackets.sub(r'\1', url)
        return url, end_idx