parse local or remote torrent file
775416
def parse_torrent_file(torrent):
    """"""
    link_re = re.compile(r'^(http?s|ftp)')
    if link_re.match(torrent):
        response = requests.get(torrent, headers=HEADERS, timeout=20)
        data = parse_torrent_buffer(response.content)
    elif os.path.isfile(torrent):
        with open(torrent, 'rb') as f:
            data = parse_torrent_buffer(f.read())
    else:
        data = None
    return data
598972
def _get_torrent(self, row):
        
        td = row("td")
        name = td("a.cellMainLink").text()
        name = name.replace(" . ", ".").replace(" .", ".")
        author = td("a.plain").text()
        verified_author = True if td(".lightgrey>.ka-verify") else False
        category = td("span").find("strong").find("a").eq(0).text()
        verified_torrent = True if td(".icon16>.ka-green") else False
        comments = td(".iaconbox>.icommentjs>.iconvalue").text()
        torrent_link = "http://" + BASE.domain
        if td("a.cellMainLink").attr("href") is not None:
            torrent_link += td("a.cellMainLink").attr("href")
        magnet_link = td("a[data-nop]").eq(1).attr("href")
        download_link = td("a[data-download]").attr("href")

        td_centers = row("td.center")
        size = td_centers.eq(0).text()
        files = td_centers.eq(1).text()
        age = " ".join(td_centers.eq(2).text().split())
        seed = td_centers.eq(3).text()
        leech = td_centers.eq(4).text()

        return Torrent(name, author, verified_author, category, size,
                       files, age, seed, leech, verified_torrent, comments,
                       torrent_link, magnet_link, download_link)
679647
def parse(self):
        """"""
        for package in os.listdir(self.root_directory):
            directory = os.path.join(self.root_directory, package)
            if not os.path.isdir(directory):
                continue
            dir_contents = os.listdir(directory)
            sdists = [tarball for tarball in dir_contents
                      if (tarball.endswith('.tar.gz')
                          and tarball.startswith(package + '-'))]
            for sdist in sdists:
                version = sdist.replace('.tar.gz', '').replace(
                    package + '-', '')
                self.packages[package].append(version)
84843
def scp_file_remote_to_local(self, remote_path, local_path):
        

        sshadd_command = [
            'ssh-add',
            '/Users/pyrat/.ssh/ubuntuNode'
        ]
        self.info_log(
            "executing command: %s" %
            ' '.join(sshadd_command)
        )
        p = subprocess.Popen(sshadd_command)
        p.wait()

        scp_command = [
            'scp',
            '-o',
            'StrictHostKeyChecking=no',
            '%s@%s:"%s"' %
            (
                self.browser_config.get('username'),
                self.get_ip(),
                remote_path
            ),
            local_path
        ]
        self.info_log(
            "executing command: %s" %
            ' '.join(scp_command)
        )
        p = subprocess.Popen(scp_command)
        p.wait()
709089
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
            knowledge_base):
    """"""
    _ = stderr, time_taken, args, knowledge_base  # Unused.
    self.CheckReturn(cmd, return_val)
    result = rdf_protodict.AttributedDict()
    for entry in self._field_parser.ParseEntries(stdout):
      line_str = " ".join(entry)
      mount_rslt = self.mount_re.match(line_str)
      if mount_rslt:
        device, mount_point, fs_type, option_str = mount_rslt.groups()
        result = rdf_client_fs.Filesystem()
        result.device = device
        result.mount_point = mount_point
        result.type = fs_type
        # Parse these options as a dict as some items may be key/values.
        # KeyValue parser uses OrderedDict as the native parser method. Use it.
        options = KeyValueParser(term=",").ParseToOrderedDict(option_str)
        # Keys without values get assigned [] by default. Because these keys are
        # actually true, if declared, change any [] values to True.
        for k, v in iteritems(options):
          options[k] = v or [True]
        result.options = rdf_protodict.AttributedDict(**options)
        yield result
386049
def getRemoteFile(urlOrPath, destPath):
    '''
    
    '''
    urlp = urlparse(urlOrPath)
    if urlp.scheme == '':
        return (os.path.abspath(urlOrPath), 'local')
    elif urlp.scheme not in ('http', 'https'):
        return (urlOrPath, 'local')
    else:
        filename = toFilename(urlOrPath)
        destPath = destPath + '/' + filename
        log.info('Retrieving %s to %s.' % (urlOrPath, destPath))
        try:
            urlretrieve(urlOrPath, destPath)
        except IOError:
            # monkey patch fix for SSL/Windows per Tika-Python #54 
            # https://github.com/chrismattmann/tika-python/issues/54
            import ssl
            if hasattr(ssl, '_create_unverified_context'):
                ssl._create_default_https_context = ssl._create_unverified_context
            # delete whatever we had there
            if os.path.exists(destPath) and os.path.isfile(destPath):
                os.remove(destPath)
            urlretrieve(urlOrPath, destPath)
        return (destPath, 'remote')
923054
def parse_content_disposition(text):
    ''''''
    match = re.search(r'filename\s*=\s*(.+)', text, re.IGNORECASE)

    if not match:
        return

    filename = match.group(1)

    if filename[0] in '"\'':
        match = re.match(r'(.)(.+)(?!\\)\1', filename)

        if match:
            filename = match.group(2).replace('\\"', '"')

            return filename

    else:
        filename = filename.partition(';')[0].strip()
        return filename
727491
function fetchTorrent(infoHash, opts, cb) {
    var engine = new torrentStream(infoHash, { 
        connections: 30,
        trackers: cfg.fetchTorrentTrackers,
    });
    var cb = _.once(cb);

    engine.ready(function() { 
        cb(null, engine.torrent);
        engine.destroy(); 
    });
    setTimeout(function() {
        cb(new Error("fetchTorrent timed out"));
        engine.destroy();
    }, cfg.fetchMetaTimeout || 25 * 1000);
}
534101
def fetch(self, is_dl_forced=False):
        

        (files_to_download, ftp) = self._get_file_list(
            self.files['anat_entity']['path'],
            self.files['anat_entity']['pattern'])

        LOG.info(
            'Will Check \n%s\nfrom %s',
            '\n'.join(list(files_to_download)), ftp.getwelcome())

        for dlname in files_to_download:
            localfile = '/'.join((self.rawdir, dlname))
            info = ftp.sendcmd("MLST {}".format(dlname))   # fetch remote file stats
            info = info.split('\n')[1].strip()              # drop pre & post script
            info = info.split(';')                          # partition fields
            info = [item.strip() for item in info[:-1]]     # cleanup an drop final name
            info = [item.split('=') for item in info]       # make pairs
            info = {item[0]: item[1] for item in info}      # transform list to dict

            LOG.info(
                '%s\n'
                'Remote File Size: %i\n'
                'Remote timestamp: %s',
                dlname, int(info['size']),
                self._convert_ftp_time_to_iso(info['modify']))

            if not os.path.exists(localfile) or is_dl_forced or \
                    self.checkIfRemoteIsNewer(
                        localfile, int(info['size']), info['modify']):

                LOG.info("Fetching %s", dlname)
                LOG.info("Writing to %s", localfile)

                ftp.retrbinary('RETR {}'.format(dlname), open(localfile, 'wb').write)
                remote_dt = Bgee._convert_ftp_time_to_iso(info['modify'])
                os.utime(
                    localfile,
                    (time.mktime(remote_dt.timetuple()),
                     time.mktime(remote_dt.timetuple())))
        ftp.quit()

        return
775159
def locally_cache_remote_file(href, dir):
    
    scheme, host, remote_path, params, query, fragment = urlparse(href)
    
    assert scheme in ('http','https'), 'Scheme must be either http or https, not "%s" (for %s)' % (scheme,href)

    head, ext = posixpath.splitext(posixpath.basename(remote_path))
    head = sub(r'[^\w\-_]', '', head)
    hash = md5(href).hexdigest()[:8]
    
    local_path = '%(dir)s/%(host)s-%(hash)s-%(head)s%(ext)s' % locals()

    headers = {}
    if posixpath.exists(local_path):
        msg('Found local file: %s' % local_path )
        t = localtime(os.stat(local_path).st_mtime)
        headers['If-Modified-Since'] = strftime('%a, %d %b %Y %H:%M:%S %Z', t)
    
    if scheme == 'https':
        conn = HTTPSConnection(host, timeout=5)
    else:
        conn = HTTPConnection(host, timeout=5)

    if query:
        remote_path += '?%s' % query

    conn.request('GET', remote_path, headers=headers)
    resp = conn.getresponse()
        
    if resp.status in range(200, 210):
        # hurrah, it worked
        f = open(un_posix(local_path), 'wb')
        msg('Reading from remote: %s' % remote_path)
        f.write(resp.read())
        f.close()

    elif resp.status in (301, 302, 303) and resp.getheader('location', False):
        # follow a redirect, totally untested.
        redirected_href = urljoin(href, resp.getheader('location'))
        redirected_path = locally_cache_remote_file(redirected_href, dir)
        os.rename(redirected_path, local_path)
    
    elif resp.status == 304:
        # hurrah, it's cached
        msg('Reading directly from local cache')
        pass

    else:
        raise Exception("Failed to get remote resource %s: %s" % (href, resp.status))
    
    return local_path
660165
def remote_list(timestamps=True):
    
    r = get_remote_file(DICTS_URL)
    lst = []
    for f in r.text.split('\n'):
        if not f:
            continue
        name, date = f.split(':')
        name = name.replace('.txt', '')
        lst.append((name, int(date)) if timestamps else name)

    return lst
886398
def download(self, torrent_id, directory, filename) :
        
        return self.call('torrents/download/%s' % torrent_id, params={'filename' : filename, 'directory' : directory})
2865
def parse(self):
        
        try:
            if not os.path.getsize(self.ns.pathname):
                # Ignore 0-byte dummy files (Firefox creates these while downloading)
                self.job.LOG.warn("Ignoring 0-byte metafile '%s'" % (self.ns.pathname,))
                return
            self.metadata = metafile.checked_open(self.ns.pathname)
        except EnvironmentError as exc:
            self.job.LOG.error("Can't read metafile '%s' (%s)" % (
                self.ns.pathname, str(exc).replace(": '%s'" % self.ns.pathname, ""),
            ))
            return
        except ValueError as exc:
            self.job.LOG.error("Invalid metafile '%s': %s" % (self.ns.pathname, exc))
            return

        self.ns.info_hash = metafile.info_hash(self.metadata)
        self.ns.info_name = self.metadata["info"]["name"]
        self.job.LOG.info("Loaded '%s' from metafile '%s'" % (self.ns.info_name, self.ns.pathname))

        # Check whether item is already loaded
        try:
            name = self.job.proxy.d.name(self.ns.info_hash, fail_silently=True)
        except xmlrpc.HashNotFound:
            pass
        except xmlrpc.ERRORS as exc:
            if exc.faultString != "Could not find info-hash.":
                self.job.LOG.error("While checking for #%s: %s" % (self.ns.info_hash, exc))
                return
        else:
            self.job.LOG.warn("Item #%s '%s' already added to client" % (self.ns.info_hash, name))
            return

        return True
1039755
def Parse(self, statentry, file_object, knowledge_base):
    """"""

    plist = biplist.readPlist(file_object)

    if not isinstance(plist, list):
      raise parser.ParseError(
          "InstallHistory plist is a '%s', expecting a list" % type(plist))

    packages = []
    for sw in plist:
      packages.append(
          rdf_client.SoftwarePackage(
              name=sw.get("displayName"),
              version=sw.get("displayVersion"),
              description=",".join(sw.get("packageIdentifiers")),
              # TODO(hanuszczak): make installed_on an RDFDatetime
              installed_on=_DateToEpoch(sw.get("date")),
              install_state=rdf_client.SoftwarePackage.InstallState.INSTALLED))

    if packages:
      yield rdf_client.SoftwarePackages(packages=packages)
342261
def ppiece(self, content):
        '''
        
        '''
        piece_index, byte_begin = struct.unpack('!ii', content[0:8])

        # TODO -- figure out a better way to catch this error.
        # How is piece_index getting swapped out from under me?
        if piece_index != self.piece.index:
            return

        assert byte_begin % REQUEST_SIZE == 0
        block_begin = byte_begin / REQUEST_SIZE
        block = content[8:]
        self.piece.save(index=block_begin, bytes=block)
        if self.piece.complete:
            piece_bytes = self.piece.get_bytes()
            if self.piece.index == self.torrent.last_piece:
                piece_bytes = piece_bytes[:self.torrent.last_piece_length]
            if hashlib.sha1(piece_bytes).digest() == (self.torrent.torrent_dict
                                                      ['info']['pieces']
                                                      [20 * piece_index:20 *
                                                       piece_index + 20]):

                print 'hash matches'

                # Take care of visualizer stuff
                piece_dict = {'kind': 'piece', 'peer': self.sock.getpeername(),
                              'piece_index': piece_index}
                self.torrent.switchboard.broadcast(piece_dict)

                print ('writing piece {}. Length is '
                       '{}').format(repr(piece_bytes)[:10] + '...',
                                    len(piece_bytes))

                # Write out
                byte_index = piece_index * self.torrent.piece_length
                self.piece = self.init_piece()
                self.request_all()
                self.torrent.switchboard.write(byte_index, piece_bytes)
                self.torrent.switchboard.mark_off(piece_index)
                print self.torrent.switchboard.bitfield
                if self.torrent.switchboard.complete:
                    print '\nDownload complete\n'
                    self.reactor.is_running = False
            else:
                print "Bad data -- hash doesn't match. Discarding piece."
                self.piece = self.init_piece()
                self.request_all()
1135832
def get_remote_file(url):
    
    try:
        return requests.get(url)
    except requests.exceptions.ConnectionError as e:
        print("Connection error!")
        print(e.message.reason)
        exit(1)
1188379
def download(self, local, remote):
        
        self.sync(RemoteFile(remote, self.api), LocalFile(local))