Scrap the number of organizations from a GitHub profile.

        :param web: parsed web.
        :type web: BeautifulSoup node.
519921
def __getOrganizations(self, web):
        
        orgsElements = web.find_all("a", {"class": "avatar-group-item"})
        self.organizations = len(orgsElements)
366231
def get_organizations(self, page=None):
        """"""
        opts = {}
        if page:
            opts['page'] = page
        return self.api_call(ENDPOINTS['organizations']['list'], **opts)
245876
def find_repo_urls(session, name, candidates):
    
    for _url in candidates:
        if validate_url(_url):
            try:
                resp = session.get(_url)
                if resp.status_code == 200:
                    tree = etree.HTML(resp.content)
                    if tree:
                        for link in frozenset([str(l) for l in tree.xpath("//a/@href")]):
                            # check if the link 1) is to github.com / bitbucket.org AND 2) somewhat
                            # contains the project name
                            if ("github.com" in link or "bitbucket.org" in link or
                                    "sourceforge.net" in link) \
                                    and contains_project_name(name, link):
                                link = validate_url(validate_repo_url(url=link))
                                if link:
                                    logger.debug("Found repo URL {}".format(link))
                                    yield link
            except ConnectionError:
                # we really don't care about connection errors here. a lot of project pages are simply
                # down because the project is no longer maintained
                pass
            except etree.XMLSyntaxError:
                # unable to parse HTML
                pass
            except UnicodeEncodeError:
                pass
660019
def webhook_url(self):
        """"""
        webhook_token = ProviderToken.query.filter_by(
            id=self.account.extra_data['tokens']['webhook']
        ).first()
        if webhook_token:
            wh_url = current_app.config.get('GITHUB_WEBHOOK_RECEIVER_URL')
            if wh_url:
                return wh_url.format(token=webhook_token.access_token)
            else:
                raise RuntimeError('You must set GITHUB_WEBHOOK_RECEIVER_URL.')
1047566
def scrape(url, params=None, user_agent=None):
    '''
    
    '''

    headers = {}

    if user_agent:
        headers['User-Agent'] = user_agent

    data = params and six.moves.urllib.parse.urlencode(params) or None
    req = six.moves.urllib.request.Request(url, data=data, headers=headers)
    f = six.moves.urllib.request.urlopen(req)

    text = f.read()
    f.close()

    return text
1282972
def get_organization_events(self, org):
        
        assert isinstance(org, github.Organization.Organization), org
        return github.PaginatedList.PaginatedList(
            github.Event.Event,
            self._requester,
            "/users/" + self.login + "/events/orgs/" + org.login,
            None
        )
427892
def get_orgs(self):
        
        return github.PaginatedList.PaginatedList(
            github.Organization.Organization,
            self._requester,
            "/user/orgs",
            None
        )
951614
function scrapeSearchPage(pageUrl) {

    return co(function *() {
        //Load the HTML page
        const $ = cheerio.load(yield request(pageUrl));

        //For each card, scrape some data
        const cards = Array.from($("#cardResults li")).map(el => {
            const $card = $(el);

            //Just scrape the database URL and the image for now
            const url = Url.resolve(SCRAPE_URL, $card.find("a").attr("href"));
            return {
                url: url,
                image: Url.resolve(SCRAPE_URL, $card.find("img").attr("src")),
                id: cardIdFromUrl(url)
            };
        });

        //Work out how many pages in total there are
        const totalText = $('#cards-load-more>div>span').text();
        const numPages = parseInt(/\d of (\d+)/.exec(totalText)[1]);

        return {
            numPages: numPages,
            cards: cards
        };

    })();
}
1248061
function parseGithubURL(url) {
  var githubUrlRe = /github\.com\/([^/]+)\/([^/]+)\/pull\/([0-9]+)/;
  var match = url.match(githubUrlRe);

  if (!match) {
    return null;
  }

  return {
    owner: match[1],
    repo: match[2],
    number: match[3]
  };
}
1145007
def validate_repo_url(url):
    
    try:
        if "github.com" in url:
            return re.findall(r"https?://w?w?w?.?github.com/[\w\-]+/[\w.-]+", url)[0]
        elif "bitbucket.org" in url:
            return re.findall(r"https?://bitbucket.org/[\w.-]+/[\w.-]+", url)[0] + "/src/"
        elif "launchpad.net" in url:
            return re.findall(r"https?://launchpad.net/[\w.-]+", url)[0]
        elif "sourceforge.net" in url:
            mo = re.match(r"https?://sourceforge.net/projects/"
                          r"([\w.-]+)/", url, re.I)
            template = "https://sourceforge.net/p/{}/code/HEAD/tree/trunk/src/"
            return template.format(mo.groups()[0])
    except (IndexError, AttributeError):
        pass
    return ""
172969
def get_pages(url):
    
    while True:
        yield url
        doc = html.parse(url).find("body")
        links = [a for a in doc.findall(".//a") if a.text and a.text.startswith("next ")]
        if not links:
            break
        url = urljoin(url, links[0].get('href'))
591100
def __getBio(self, web):
        
        bio = web.find_all("div", {"class": "user-profile-bio"})

        if bio:
            try:
                bio = bio[0].text
                if bio and GitHubUser.isASCII(bio):
                    bioText = bio.replace("\n", "")
                    bioText = bioText.replace("\t", " ").replace("\"", "")
                    bioText = bioText.replace("\'", "").replace("\\", "")
                    self.bio = bioText
                else:
                    self.bio = ""
            except IndexError as error:
                print("There was an error with the user " + self.name)
                print(error)
            except AttributeError as error:
                print("There was an error with the user " + self.name)
                print(error)
337868
def followee_num(self):
        
        if self.url is None:
            return 0
        else:
            number = int(self.soup.find(
                'div', class_='zm-profile-side-following').a.strong.text)
            return number
133407
def fetch(url: str, **kwargs) -> Selector:
    
    kwargs.setdefault('headers', DEFAULT_HEADERS)
    try:
        res = requests.get(url, **kwargs)
        res.raise_for_status()
    except requests.RequestException as e:
        print(e)
    else:
        html = res.text
        tree = Selector(text=html)
        return tree
878327
def __parse_organizations(self, domain_to_employer):
        """"""

        # Parse streams
        self.__parse_domain_to_employer_stream(domain_to_employer)

        for org in self.__raw_orgs:
            o = Organization(name=org)

            for dom in self.__raw_orgs[org]:
                d = Domain(domain=dom, is_top_domain=False)
                o.domains.append(d)

            self._organizations[org] = o
547802
def get_json(uri):
    
    response = requests.get(API + uri, headers=HEADERS)

    limit = int(response.headers.get("x-ratelimit-remaining"))
    if limit == 0:
        sys.stdout.write("\n")
        message = "You have run out of GitHub request tokens. "

        if int(response.headers.get("x-ratelimit-limit")) == 60:
            message += "Set a GITHUB_TOKEN to increase your limit to 5000/hour. "

        wait_seconds = int(response.headers.get("x-ratelimit-reset")) - int(time.time())
        wait_minutes = math.ceil(wait_seconds / 60)
        message += "Try again in ~%d minutes. " % wait_minutes

        if "--wait-for-reset" in sys.argv:
            progress_message(message.replace("Try ", "Trying "))
            time.sleep(wait_seconds + 1)
            progress_message("Resuming")
            return get_json(uri)
        else:
            raise ValueError(message)

    progress()
    return response.json()
209170
def list(self):
        
        # Define the basic request. The per_page parameter is set to 100, which
        # is the maximum github allows. If the user has more than one page of
        # gists, this request object will be modified to retrieve each
        # successive page of gists.
        request = requests.Request(
                'GET',
                'https://api.github.com/gists',
                headers={
                    'Accept-Encoding': 'identity, deflate, compress, gzip',
                    'User-Agent': 'python-requests/1.2.0',
                    'Accept': 'application/vnd.github.v3.base64',
                    },
                params={
                    'access_token': self.token,
                    'per_page': 100,
                    },
                )

        # Github provides a 'link' header that contains information to
        # navigate through a users page of gists. This regex is used to
        # extract the URLs contained in this header, and to find the next page
        # of gists.
        pattern = re.compile(r'<([^>]*)>; rel="([^"]*)"')

        gists = []
        while True:

            # Retrieve the next page of gists
            try:
                response = self.send(request).json()

            except Exception:
                break

            # Extract the list of gists
            for gist in response:
                try:
                    gists.append(
                            GistInfo(
                                gist['id'],
                                gist['public'],
                                gist['description'],
                                )
                            )

                except KeyError:
                    continue

            try:
                link = response.headers['link']

                # Search for the next page of gist. If a 'next' page is found,
                # the URL is set to this new page and the iteration continues.
                # If there is no next page, return the list of gists.
                for result in pattern.finditer(link):
                    url = result.group(1)
                    rel = result.group(2)
                    if rel == 'next':
                        request.url = url
                        break
                else:
                    return gists

            except Exception:
                break

        return gists