Gets temporary S3 credentials to stage user-uploaded files
511317
def _get_credentials(self):
        
        uri = URITemplate(self.baseuri + '/{username}/credentials').expand(
            username=self.username)

        resp = self.session.post(uri)

        self.handle_http_error(
            resp,
            custom_messages={
                401: "Token is not authorized",
                404: "Token does not have upload scope",
                429: "Too many requests"})
        return resp
424095
def credential_file(file=nil)
      unless file.nil?
        dsl_options[:credential_file]=file
        dsl_options.merge!(Ec2.load_keys_from_credential_file(file))
      else
        fetch(:credential_file)
      end
    end
171646
def get_credentials_for_url(url, opts, force_user=None):
    
    creds = None
    verbose = int(opts.get("verbose"))
    force_prompt = opts.get("prompt", False)
    allow_prompt = not opts.get("no_prompt", True)
    allow_keyring = not opts.get("no_keyring", False) and not force_user
    allow_netrc = not opts.get("no_netrc", False) and not force_user

    # print("get_credentials_for_url", force_user, allow_prompt)
    if force_user and not allow_prompt:
        raise RuntimeError(
            "Cannot get credentials for a distinct user ({}) from keyring or .netrc and "
            "prompting is disabled.".format(force_user)
        )

    # Lookup our own pyftpsync 1.x credential store. This is deprecated with 2.x
    home_path = os.path.expanduser("~")
    file_path = os.path.join(home_path, DEFAULT_CREDENTIAL_STORE)
    if os.path.isfile(file_path):
        raise RuntimeError(
            "Custom password files are no longer supported. Delete {} and use .netrc instead.".format(
                file_path
            )
        )

    # Query keyring database
    if creds is None and keyring and allow_keyring:
        try:
            # Note: we pass the url as `username` and username:password as `password`
            c = keyring.get_password("pyftpsync", url)
            if c is not None:
                creds = c.split(":", 1)
                write(
                    "Using credentials from keyring('pyftpsync', '{}'): {}:***.".format(
                        url, creds[0]
                    )
                )
            else:
                if verbose >= 4:
                    write(
                        "No credentials found in keyring('pyftpsync', '{}').".format(
                            url
                        )
                    )
        #        except keyring.errors.TransientKeyringError:
        except Exception as e:
            # e.g. user clicked 'no'
            write_error("Could not get password from keyring {}".format(e))

    # Query .netrc file
    #     print(opts)
    if creds is None and allow_netrc:
        try:
            authenticators = None
            authenticators = netrc.netrc().authenticators(url)
        except CompatFileNotFoundError:
            if verbose >= 4:
                write("Could not get password (no .netrc file).")
        except Exception as e:
            write_error("Could not read .netrc: {}.".format(e))

        if authenticators:
            creds = (authenticators[0], authenticators[2])
            write("Using credentials from .netrc file: {}:***.".format(creds[0]))
        else:
            if verbose >= 4:
                write("Could not find entry for '{}' in .netrc file.".format(url))

    # Prompt for password if we don't have credentials yet, or --prompt was set.
    if allow_prompt:
        if creds is None:
            creds = prompt_for_password(url)
        elif force_prompt:
            # --prompt was set but we can provide a default for the user name
            creds = prompt_for_password(url, default_user=creds[0])

    return creds
596521
def upload_bam_to_s3(job, job_vars):
    
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    # I/O
    job.fileStore.readGlobalFile(ids['alignments.bam'], os.path.join(work_dir, 'alignments.bam'))
    bam_path = os.path.join(work_dir, 'alignments.bam')
    sample_name = uuid + '.bam'
    # Parse s3_dir to get bucket and s3 path
    s3_dir = input_args['s3_dir']
    bucket_name = s3_dir.split('/')[0]
    bucket_dir = os.path.join('/'.join(s3_dir.split('/')[1:]), 'bam_files')
    # Upload to S3 via S3AM
    s3am_command = ['s3am',
                    'upload',
                    'file://{}'.format(bam_path),
                    os.path.join('s3://', bucket_name, bucket_dir, sample_name)]
    subprocess.check_call(s3am_command)
1033048
def authenticate_storage(client_secrets, read_only=False):
    """"""
    if read_only:
        scopes = ['https://www.googleapis.com/auth/devstorage.read_only']
    else:
        scopes = ['https://www.googleapis.com/auth/devstorage.read_write']

    credentials = service_account.Credentials.from_service_account_info(
            client_secrets)
    
    scoped_credentials = credentials.with_scopes(scopes)
    
    return scoped_credentials
514040
def copy_s3_bucket(src_bucket_name, src_bucket_secret_key, src_bucket_access_key,
                   dst_bucket_name, dst_bucket_secret_key, dst_bucket_access_key):
    
    with cd(env.remote_path):
        tmp_dir = "s3_tmp"
        sudo('rm -rf %s' % tmp_dir, warn_only=True, user=env.remote_user)
        sudo('mkdir %s' % tmp_dir, user=env.remote_user)
        sudo('s3cmd --recursive get s3://%s/upload/ %s --secret_key=%s --access_key=%s' % (
            src_bucket_name, tmp_dir, src_bucket_secret_key, src_bucket_access_key),
            user=env.remote_user)
        sudo('s3cmd --recursive put %s/ s3://%s/upload/ --secret_key=%s --access_key=%s' % (
            tmp_dir, dst_bucket_name, dst_bucket_secret_key, dst_bucket_access_key),
            user=env.remote_user)

        sudo('s3cmd setacl s3://%s/upload --acl-public --recursive --secret_key=%s --access_key=%s' % (
             dst_bucket_name, dst_bucket_secret_key, dst_bucket_access_key),
             user=env.remote_user)
        # cleanup
        sudo('rm -rf %s' % tmp_dir, warn_only=True, user=env.remote_user)
479487
@Override
    public AWSCredentials getCredentials() {
        if (SDKGlobalConfiguration.isEc2MetadataDisabled()) {
            throw new AmazonClientException("AWS_EC2_METADATA_DISABLED is set to true, not loading credentials from EC2 Instance "
                                         + "Metadata service");
        }
        AWSCredentials creds = credentialsFetcher.getCredentials();
        shouldRefresh = true;
        return creds;
    }
1127264
def temporary_url(expires_at = Time.now + 3600)
      url = URI.escape("#{protocol}#{host(true)}/#{path_prefix}#{key}")
      signature = Signature.generate_temporary_url_signature(:bucket => name,
                                                             :resource => key,
                                                             :expires_at => expires_at,
                                                             :secret_access_key => secret_access_key)

      "#{url}?SNDAAccessKeyId=#{access_key_id}&Expires=#{expires_at.to_i.to_s}&Signature=#{signature}"
    end
873917
def temp_url(self, duration=120):
        """"""
        return self.bucket._boto_s3.meta.client.generate_presigned_url(
            'get_object',
            Params={'Bucket': self.bucket.name, 'Key': self.name},
            ExpiresIn=duration
        )
323797
def _setup_s3(settings, path, prompt=True):
    
    ret = {'default_s3_buckets': {}, 's3_credentials': settings.get('s3_credentials', {})}

    if prompt:
        use = raw_input("\nWould you like to set up Amazon S3? [Y/n] ")
        if use.lower() != "y" and use != "":
            puts("\n- Not configuring Amazon S3.")
            return ret

    existing_access_key = settings.get('default_s3_access_key_id', None) or \
                          os.environ.get('AWS_ACCESS_KEY_ID', None)
    existing_secret_key = settings.get('default_s3_secret_access_key', None) or \
                          os.environ.get('AWS_SECRET_ACCESS_KEY', None)

    access_key_prompt = "\nPlease enter your default Amazon Access Key ID:"
    if existing_access_key:
        access_key_prompt += ' [%s] ' % existing_access_key
    else:
        access_key_prompt += ' (leave blank to skip) '
    default_aws_access_key_id = raw_input(access_key_prompt)

    if default_aws_access_key_id == '' and existing_access_key:
        default_aws_access_key_id = existing_access_key


    if default_aws_access_key_id:
        secret_key_prompt = "\nPlease enter your default Amazon Secret Access Key:"
        if existing_secret_key:
            secret_key_prompt += ' [%s] ' % existing_secret_key
        else:
            secret_key_prompt += ' (leave blank to skip) '
        default_aws_secret_access_key = raw_input(secret_key_prompt)

        if default_aws_secret_access_key == '' and existing_secret_key:
            default_aws_secret_access_key = existing_secret_key

        ret.update({
            'default_s3_access_key_id': default_aws_access_key_id,
            'default_s3_secret_access_key': default_aws_secret_access_key,
        })

    # If we're all set with AWS creds, we can setup our default
    # staging and production buckets
    if default_aws_access_key_id and default_aws_secret_access_key:
        existing_staging_bucket = None
        existing_production_bucket = None
        if settings.get('default_s3_buckets'):
            existing_staging_bucket = settings['default_s3_buckets'].get('staging', None)
            existing_production_bucket = settings['default_s3_buckets'].get('production', None)

        staging_prompt = "\nWhat is your default staging bucket?"
        if existing_staging_bucket:
            staging_prompt += ' [%s] ' % existing_staging_bucket
        else:
            staging_prompt += ' (e.g. apps.beta.myorg.com, leave blank to skip) '
        staging = raw_input(staging_prompt)

        if staging == '' and existing_staging_bucket:
            staging = existing_staging_bucket
        if staging != "":
            ret['default_s3_buckets'].update({
                'staging': staging,
            })

        production_prompt = "\nWhat is your default production bucket?"
        if existing_production_bucket:
            production_prompt += ' [%s] ' % existing_production_bucket
        else:
            production_prompt += ' (e.g. apps.myorg.com, leave blank to skip) '
        production = raw_input(production_prompt)

        if production == '' and existing_production_bucket:
            production = existing_production_bucket
        if production != "":
            ret['default_s3_buckets'].update({
                'production': production,
            })


    more_prompt = "\nWould you like to add additional buckets and credentials? [y/N] "
    while raw_input(more_prompt).lower() == 'y':
        ## Ask for a uri
        additional_s3_bucket = raw_input(
            "\nPlease specify an additional bucket (e.g. "
            "additional.bucket.myorg.com/, leave blank to skip adding bucket) ")
        if additional_s3_bucket == "":
            continue

        ## Ask for an access key, if it differs from the default
        additional_access_key_prompt = "\nPlease specify an AWS Access Key ID for this bucket:"

        if default_aws_access_key_id:
            additional_access_key_prompt += ' [%s] ' % default_aws_access_key_id
        else:
            additional_access_key_prompt += ' (leave blank to skip adding bucket) '

        additional_aws_access_key_id = raw_input(additional_access_key_prompt)

        if additional_aws_access_key_id == "" and default_aws_access_key_id:
            additional_aws_access_key_id = default_aws_access_key_id
        elif additional_aws_access_key_id == "":
            continue

        # Ask for a secret key, if it differs from default
        additional_secret_key_prompt = "\nPlease specify an AWS Secret Access Key for this bucket:"

        if default_aws_secret_access_key:
            additional_secret_key_prompt += ' [%s] ' % default_aws_secret_access_key
        else:
            additional_secret_key_prompt += ' (leave blank to skip adding bucket) '

        additional_aws_secret_access_key = raw_input(
            additional_secret_key_prompt)

        if additional_aws_secret_access_key == "" and default_aws_secret_access_key:
            additional_aws_secret_access_key = default_aws_secret_access_key
        elif additional_aws_secret_access_key == "":
            continue

        ret['s3_credentials'][additional_s3_bucket] = {
            'access_key_id': additional_aws_access_key_id,
            'secret_access_key': additional_aws_secret_access_key,
        }

    puts("\n- Done configuring Amazon S3.")
    return ret
141949
def read_creds_from_aws_credentials_file(profile_name, credentials_file = aws_credentials_file):
    
    credentials = init_creds()
    profile_found = False
    try:
        # Make sure the ~.aws folder exists
        if not os.path.exists(aws_config_dir):
            os.makedirs(aws_config_dir)
        with open(credentials_file, 'rt') as cf:
            for line in cf:
                profile_line = re_profile_name.match(line)
                if profile_line:
                    if profile_line.groups()[0] == profile_name:
                        profile_found = True
                    else:
                        profile_found = False
                if profile_found:
                    if re_access_key.match(line):
                        credentials['AccessKeyId'] = line.split("=")[1].strip()
                    elif re_secret_key.match(line):
                        credentials['SecretAccessKey'] = line.split("=")[1].strip()
                    elif re_mfa_serial.match(line):
                        credentials['SerialNumber'] = (line.split('=')[1]).strip()
                    elif re_session_token.match(line) or re_security_token.match(line):
                        credentials['SessionToken'] = ('='.join(x for x in line.split('=')[1:])).strip()
                    elif re_expiration.match(line):
                        credentials['Expiration'] = ('='.join(x for x in line.split('=')[1:])).strip()
    except Exception as e:
        # Silent if error is due to no ~/.aws/credentials file
        if not hasattr(e, 'errno') or e.errno != 2:
            printException(e)
    return credentials
562725
function getUserMetadataFileName(options, user) {
	let userId;
	if(typeof user === "string") {
		userId = user;
	} else if(user && typeof user.username === 'string') {
		userId = user.username;
	} else {
		return null;
	}
	return nodePath.join(getUserRootLocation(options, userId), FILENAME_USER_METADATA);
}
1363741
def fetch_cluster_credentials(cls, db_user, db_name, cluster_id,
                                  duration_s):
        """"""
        boto_client = boto3.client('redshift')

        try:
            return boto_client.get_cluster_credentials(
                DbUser=db_user,
                DbName=db_name,
                ClusterIdentifier=cluster_id,
                DurationSeconds=duration_s,
                AutoCreate=False)

        except boto_client.exceptions.ClientError as e:
            raise dbt.exceptions.FailedToConnectException(
                "Unable to get temporary Redshift cluster credentials: {}"
                .format(e))
251520
def _refresh_buckets_cache_file(creds, cache_file, multiple_env, environment, prefix):
    '''
    
    '''

    # helper s3 query function
    def __get_s3_meta():
        return __utils__['s3.query'](
            key=creds.key,
            keyid=creds.keyid,
            kms_keyid=creds.kms_keyid,
            bucket=creds.bucket,
            service_url=creds.service_url,
            verify_ssl=creds.verify_ssl,
            location=creds.location,
            return_bin=False,
            params={'prefix': prefix},
            path_style=creds.path_style,
            https_enable=creds.https_enable)

    # grab only the files/dirs in the bucket
    def __get_pillar_files_from_s3_meta(s3_meta):
        return [k for k in s3_meta if 'Key' in k]

    # pull out the environment dirs (e.g. the root dirs)
    def __get_pillar_environments(files):
        environments = [(os.path.dirname(k['Key']).split('/', 1))[0] for k in files]
        return set(environments)

    log.debug('Refreshing S3 buckets pillar cache file')

    metadata = {}
    bucket = creds.bucket

    if not multiple_env:
        # Single environment per bucket
        log.debug('Single environment per bucket mode')

        bucket_files = {}
        s3_meta = __get_s3_meta()

        # s3 query returned something
        if s3_meta:
            bucket_files[bucket] = __get_pillar_files_from_s3_meta(s3_meta)

            metadata[environment] = bucket_files

    else:
        # Multiple environments per buckets
        log.debug('Multiple environment per bucket mode')
        s3_meta = __get_s3_meta()

        # s3 query returned data
        if s3_meta:
            files = __get_pillar_files_from_s3_meta(s3_meta)
            environments = __get_pillar_environments(files)

            # pull out the files for the environment
            for saltenv in environments:
                # grab only files/dirs that match this saltenv.
                env_files = [k for k in files if k['Key'].startswith(saltenv)]

                if saltenv not in metadata:
                    metadata[saltenv] = {}

                if bucket not in metadata[saltenv]:
                    metadata[saltenv][bucket] = []

                metadata[saltenv][bucket] += env_files

    # write the metadata to disk
    if os.path.isfile(cache_file):
        os.remove(cache_file)

    log.debug('Writing S3 buckets pillar cache file')

    with salt.utils.files.fopen(cache_file, 'w') as fp_:
        pickle.dump(metadata, fp_)

    return metadata
763964
function deployS3(config) {
  if (!config.credentials.accessKeyId || !config.credentials.secretAccessKey) {
    gutil.log(
      gutil.colors.red('AWS S3 credentials missing, skipping upload...'),
    );
    return;
  }

  // create a new publisher
  const publisher = aws.create(config.credentials);

  // define custom headers
  const headers = {
    'Cache-Control': 'max-age=315360000, no-transform, public',
  };
  return (
    gulp
      .src(config.assetsPath)
      .pipe(
        rename(function(filepath) {
          filepath.dirname = path.join(config.dirname, filepath.dirname);
        }),
      )
      // gzip, Set Content-Encoding headers and add .gz extension
      .pipe(gulpIf(file => !file.path.match(/\.(mp4|ogg|webm|avi|mov|mkv)$/), aws.gzip()))
      // publisher will add Content-Length, Content-Type and headers specified above
      // If not specified it will set x-amz-acl to public-read by default
      .pipe(publisher.publish(headers))
      // create a cache file to speed up consecutive uploads
      .pipe(publisher.cache())
      // print upload updates to console
      .pipe(aws.reporter())
  );
}
1033465
def upload_to_s3(self, key, filename):
        
        extra_args = {'ACL': self.acl}
        # determine the mimetype of the file
        guess = mimetypes.guess_type(filename)
        content_type = guess[0]
        encoding = guess[1]

        if content_type:
            extra_args['ContentType'] = content_type

        # add the gzip headers, if necessary
        if (self.gzip and content_type in self.gzip_content_types) or encoding == 'gzip':
            extra_args['ContentEncoding'] = 'gzip'

        # add the cache-control headers if necessary
        if content_type in self.cache_control:
            extra_args['CacheControl'] = ''.join((
                'max-age=',
                str(self.cache_control[content_type])
            ))

        # access and write the contents from the file
        if not self.dry_run:
            logger.debug("Uploading %s" % filename)
            if self.verbosity > 0:
                self.stdout.write("Uploading %s" % filename)
            s3_obj = self.s3_resource.Object(self.aws_bucket_name, key)
            s3_obj.upload_file(filename, ExtraArgs=extra_args)

        # Update counts
        self.uploaded_files += 1
        self.uploaded_file_list.append(filename)
235554
def cli(env, identifier):
    """"""

    mgr = SoftLayer.ObjectStorageManager(env.client)
    credential_list = mgr.list_credential(identifier)
    table = formatting.Table(['id', 'password', 'username', 'type_name'])

    for credential in credential_list:
        table.add_row([
            credential['id'],
            credential['password'],
            credential['username'],
            credential['type']['name']
        ])

    env.fout(table)