Download and validate a specified Amazon PEM file.
353022
def _get_certificate(cert_url):
    """"""
    global _cache

    if cert_url in _cache:
        cert = _cache[cert_url]
        if cert.has_expired():
            _cache = {}
        else:
            return cert

    url = urlparse(cert_url)
    host = url.netloc.lower()
    path = posixpath.normpath(url.path)

    # Sanity check location so we don't get some random person's cert.
    if url.scheme != 'https' or \
       host not in ['s3.amazonaws.com', 's3.amazonaws.com:443'] or \
       not path.startswith('/echo.api/'):
        log.error('invalid cert location %s', cert_url)
        return

    resp = urlopen(cert_url)
    if resp.getcode() != 200:
        log.error('failed to download certificate')
        return

    cert = crypto.load_certificate(crypto.FILETYPE_PEM, resp.read())

    if cert.has_expired() or cert.get_subject().CN != 'echo-api.amazon.com':
        log.error('certificate expired or invalid')
        return

    _cache[cert_url] = cert
    return cert
1153415
def _serve_asset(self, path, gzipped_asset_bytes, request):
    """"""
    mimetype = mimetypes.guess_type(path)[0] or 'application/octet-stream'
    return http_util.Respond(
        request, gzipped_asset_bytes, mimetype, content_encoding='gzip')
111446
def _do_download_file(self, dss_file, fh, num_retries, min_delay_seconds):
        
        hasher = hashlib.sha256()
        delay = min_delay_seconds
        retries_left = num_retries
        while True:
            try:
                response = self.get_file._request(
                    dict(uuid=dss_file.uuid, version=dss_file.version, replica=dss_file.replica),
                    stream=True,
                    headers={
                        'Range': "bytes={}-".format(fh.tell())
                    },
                )
                try:
                    if not response.ok:
                        logger.error("%s", "File {}: GET FAILED.".format(dss_file.uuid))
                        logger.error("%s", "Response: {}".format(response.text))
                        break

                    consume_bytes = int(fh.tell())
                    server_start = 0
                    content_range_header = response.headers.get('Content-Range', None)
                    if content_range_header is not None:
                        cre = re.compile("bytes (\d+)-(\d+)")
                        mo = cre.search(content_range_header)
                        if mo is not None:
                            server_start = int(mo.group(1))

                    consume_bytes -= server_start
                    assert consume_bytes >= 0
                    if server_start > 0 and consume_bytes == 0:
                        logger.info("%s", "File {}: Resuming at {}.".format(
                            dss_file.uuid, server_start))
                    elif consume_bytes > 0:
                        logger.info("%s", "File {}: Resuming at {}. Dropping {} bytes to match".format(
                            dss_file.uuid, server_start, consume_bytes))

                        while consume_bytes > 0:
                            bytes_to_read = min(consume_bytes, 1024*1024)
                            content = response.iter_content(chunk_size=bytes_to_read)
                            chunk = next(content)
                            if chunk:
                                consume_bytes -= len(chunk)

                    for chunk in response.iter_content(chunk_size=1024*1024):
                        if chunk:
                            fh.write(chunk)
                            hasher.update(chunk)
                            retries_left = min(retries_left + 1, num_retries)
                            delay = max(delay / 2, min_delay_seconds)
                    break
                finally:
                    response.close()
            except (ChunkedEncodingError, ConnectionError, ReadTimeout):
                if retries_left > 0:
                    logger.info("%s", "File {}: GET FAILED. Attempting to resume.".format(dss_file.uuid))
                    time.sleep(delay)
                    delay *= 2
                    retries_left -= 1
                    continue
                raise
        return hasher.hexdigest()
497258
def download(config, account, day, region, output):
    """"""

    with open(config) as fh:
        config = yaml.safe_load(fh.read())

    jsonschema.validate(config, CONFIG_SCHEMA)

    found = None
    for info in config['accounts']:
        if info['name'] == account:
            found = info
            break

    if not found:
        log.info("Account %s not found", account)
        return

    s3 = boto3.client('s3')
    day = parse_date(day)

    key_data = dict(found)
    key_data['region'] = region
    key_data['date_fmt'] = "%s/%s/%s" % (
        day.year, day.month, day.day)
    key = config['key_template'] % key_data

    s3.download_file(found['bucket'], key, output + '.bz2')
    subprocess.check_call(["lbzip2", "-d", output + '.bz2'])
169337
def download_file_helper(url, input_path):
    
    r = requests.get(url, stream=True)
    if r.status_code != 200:
        cli_log.error("Failed to download file: %s" % r.json()["message"])
    local_full_path = get_download_dest(input_path, r.url)
    original_filename = os.path.split(local_full_path)[-1]
    with open(local_full_path, "wb") as f:
        click.echo("Downloading {}".format(original_filename), err=True)
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()
    pprint("Successfully downloaded %s to %s" % (original_filename, local_full_path), True)
1123802
def upload_file(ctx, opts, owner, repo, filepath, skip_errors, md5_checksum):
    """"""
    filename = click.format_filename(filepath)
    basename = os.path.basename(filename)

    click.echo(
        "Requesting file upload for %(filename)s ... "
        % {"filename": click.style(basename, bold=True)},
        nl=False,
    )

    context_msg = "Failed to request file upload!"
    with handle_api_exceptions(
        ctx, opts=opts, context_msg=context_msg, reraise_on_error=skip_errors
    ):
        with maybe_spinner(opts):
            identifier, upload_url, upload_fields = request_file_upload(
                owner=owner, repo=repo, filepath=filename, md5_checksum=md5_checksum
            )

    click.secho("OK", fg="green")

    context_msg = "Failed to upload file!"
    with handle_api_exceptions(ctx, opts=opts, context_msg=context_msg):
        filesize = utils.get_file_size(filepath=filename)

        label = "Uploading %(filename)s:" % {
            "filename": click.style(basename, bold=True)
        }

        with click.progressbar(
            length=filesize,
            label=label,
            fill_char=click.style("#", fg="green"),
            empty_char=click.style("-", fg="red"),
        ) as pb:

            def progress_callback(monitor):
                pb.update(monitor.bytes_read)

            api_upload_file(
                upload_url=upload_url,
                upload_fields=upload_fields,
                filepath=filename,
                callback=progress_callback,
            )

    return identifier
225528
function validateChecksum(requester, params) {
        params = params || {
            filePath: installerPath,
            expectedChecksum: _updateParams.checksum
        };

        var hash = crypto.createHash('sha256'),
            currentRequester = requester || "";

        if (fs.existsSync(params.filePath)) {
            var stream = fs.createReadStream(params.filePath);

            stream.on('data', function (data) {
                hash.update(data);
            });

            stream.on('end', function () {
                var calculatedChecksum = hash.digest('hex'),
                    isValidChecksum = (params.expectedChecksum === calculatedChecksum),
                    status;

                if (isValidChecksum) {
                    if (process.platform === "darwin") {
                        status = {
                            valid: true,
                            installerPath: installerPath,
                            logFilePath: logFilePath,
                            installStatusFilePath: installStatusFilePath
                        };
                    } else if (process.platform === "win32") {
                        status = {
                            valid: true,
                            installerPath: quoteAndConvert(installerPath, true),
                            logFilePath: quoteAndConvert(logFilePath, true),
                            installStatusFilePath: installStatusFilePath
                        };
                    }
                } else {
                    status = {
                        valid: false,
                        err: nodeErrorMessages.CHECKSUM_DID_NOT_MATCH
                    };
                }
                postMessageToBrackets(MessageIds.NOTIFY_VALIDATION_STATUS, currentRequester, status);
            });
        } else {
            var status = {
                valid: false,
                err: nodeErrorMessages.INSTALLER_NOT_FOUND
            };
            postMessageToBrackets(MessageIds.NOTIFY_VALIDATION_STATUS, currentRequester, status);
        }
    }
743926
def _cert_file(name, cert_type):
    '''
    
    '''
    return os.path.join(LE_LIVE, name, '{0}.pem'.format(cert_type))
1196307
def download_file(pk):
    
    release_file = models.ReleaseFile.objects.get(pk=pk)
    logging.info("Downloading %s", release_file.url)

    proxies = None
    if settings.LOCALSHOP_HTTP_PROXY:
        proxies = settings.LOCALSHOP_HTTP_PROXY
    response = requests.get(release_file.url, stream=True, proxies=proxies)

    # Write the file to the django file field
    filename = os.path.basename(release_file.url)

    # Setting the size manually since Django can't figure it our from
    # the raw HTTPResponse
    if 'content-length' in response.headers:
        size = int(response.headers['content-length'])
    else:
        size = len(response.content)

    # Setting the content type by first looking at the response header
    # and falling back to guessing it from the filename
    default_content_type = 'application/octet-stream'
    content_type = response.headers.get('content-type')
    if content_type is None or content_type == default_content_type:
        content_type = mimetypes.guess_type(filename)[0] or default_content_type

    # Using Django's temporary file upload system to not risk memory
    # overflows
    with TemporaryUploadedFile(name=filename, size=size, charset='utf-8',
                               content_type=content_type) as temp_file:
        temp_file.write(response.content)
        temp_file.seek(0)

        # Validate the md5 hash of the downloaded file
        md5_hash = md5_hash_file(temp_file)
        if md5_hash != release_file.md5_digest:
            logging.error("MD5 hash mismatch: %s (expected: %s)" % (
                md5_hash, release_file.md5_digest))
            return

        release_file.distribution.save(filename, temp_file)
        release_file.save()
    logging.info("Complete")
817725
def savePkeyPem(self, pkey, path):
        '''
        
        '''
        with s_common.genfile(path) as fd:
            fd.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
1287764
def _read_pem_file(cls, filename, from_package=False):
        '''
        '''
        _logger.debug('Reading PEM {0}.'.format(filename))

        if from_package:
            return wpull.util.filter_pem(wpull.util.get_package_data(filename))

        with open(filename, 'rb') as in_file:
            return wpull.util.filter_pem(in_file.read())
1096668
def do_verify(marfile, keyfiles=None):
    """"""
    try:
        with open(marfile, 'rb') as f:
            with MarReader(f) as m:
                # Check various parts of the mar file
                # e.g. signature algorithms and additional block sections
                errors = m.get_errors()
                if errors:
                    print("File is not well formed: {}".format(errors))
                    sys.exit(1)

                if keyfiles:
                    try:
                        keys = get_keys(keyfiles, m.signature_type)
                    except ValueError as e:
                        print(e)
                        sys.exit(1)

                    if any(m.verify(key) for key in keys):
                        print("Verification OK")
                        return True
                    else:
                        print("Verification failed")
                        sys.exit(1)
                else:
                    print("Verification OK")
                    return True

    except Exception as e:
        print("Error opening or parsing file: {}".format(e))
        sys.exit(1)
1145140
private X509Certificate loadX509FromPEMFile(String filename) {
        try {
            return (X509Certificate) CertificateFactory.getInstance("X.509").generateCertificate(FileReader.openStreamToFileFromClassPathOrPath(filename));
        } catch (Exception e) {
            throw new RuntimeException("Exception reading X509 from PEM file", e);
        }
    }
197932
def request_file_upload(owner, repo, filepath, md5_checksum=None):
    """"""
    client = get_files_api()
    md5_checksum = md5_checksum or calculate_file_md5(filepath)

    with catch_raise_api_exception():
        data, _, headers = client.files_create_with_http_info(
            owner=owner,
            repo=repo,
            data={"filename": os.path.basename(filepath), "md5_checksum": md5_checksum},
        )

    # pylint: disable=no-member
    # Pylint detects the returned value as a tuple
    ratelimits.maybe_rate_limit(client, headers)
    return data.identifier, data.upload_url, data.upload_fields
1345148
def download_file(client, bucket, remote_path, local_path, overwrite=False):
    """"""
    bucket = client.get_bucket(bucket)
    if (not overwrite) and os.path.isfile(local_path):
        raise OSError('File already exists!')
    with open(local_path, 'wb') as ofh:
        bucket.get_blob(remote_path).download_to_file(ofh)
926785
def download_rec(session, rec_url, target_path):
    """"""
    try:
        resp = session.get(session.auth.base_url + rec_url)
    except RequestException:
        raise VooblyError('failed to connect for download')
    try:
        downloaded = zipfile.ZipFile(io.BytesIO(resp.content))
        downloaded.extractall(target_path)
    except zipfile.BadZipFile:
        raise VooblyError('invalid zip file')
    return downloaded.namelist()[0]
1003906
def download(url):
    """"""
    session = requests.Session()
    session.mount('file://', FileAdapter())

    try:
        res = session.get(url)
    except requests.exceptions.ConnectionError as e:
        raise e

    res.raise_for_status()

    return res