Upload file hdfsName from hdfs to s3
512859
def upload_data(job, master_ip, inputs, hdfs_name, upload_name, spark_on_toil):
    

    if mock_mode():
        truncate_file(master_ip, hdfs_name, spark_on_toil)

    log.info("Uploading output BAM %s to %s.", hdfs_name, upload_name)
    call_conductor(job, master_ip, hdfs_name, upload_name, memory=inputs.memory)
    remove_file(master_ip, hdfs_name, spark_on_toil)
736537
def copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient, override=True):
    ''''''
    if not hdfsClient.exists(hdfsFilePath):
        raise Exception('HDFS file {} does not exist!'.format(hdfsFilePath))
    try: 
        file_status = hdfsClient.get_file_status(hdfsFilePath)
        if file_status.type != 'FILE':
            raise Exception('HDFS file path {} is not a file'.format(hdfsFilePath))
    except Exception as exception:
        nni_log(LogType.Error, 'Get hdfs file {0} status error: {1}'.format(hdfsFilePath, str(exception)))
        raise exception

    if os.path.exists(localFilePath) and override:
        os.remove(localFilePath)
    try:
        hdfsClient.copy_to_local(hdfsFilePath, localFilePath)
    except Exception as exception:
        nni_log(LogType.Error, 'Copy hdfs file {0} to {1} error: {2}'.format(hdfsFilePath, localFilePath, str(exception)))
        raise exception
    nni_log(LogType.Info, 'Successfully copied hdfs file {0} to {1}, {2} bytes'.format(hdfsFilePath, localFilePath, file_status.length))
514040
def copy_s3_bucket(src_bucket_name, src_bucket_secret_key, src_bucket_access_key,
                   dst_bucket_name, dst_bucket_secret_key, dst_bucket_access_key):
    
    with cd(env.remote_path):
        tmp_dir = "s3_tmp"
        sudo('rm -rf %s' % tmp_dir, warn_only=True, user=env.remote_user)
        sudo('mkdir %s' % tmp_dir, user=env.remote_user)
        sudo('s3cmd --recursive get s3://%s/upload/ %s --secret_key=%s --access_key=%s' % (
            src_bucket_name, tmp_dir, src_bucket_secret_key, src_bucket_access_key),
            user=env.remote_user)
        sudo('s3cmd --recursive put %s/ s3://%s/upload/ --secret_key=%s --access_key=%s' % (
            tmp_dir, dst_bucket_name, dst_bucket_secret_key, dst_bucket_access_key),
            user=env.remote_user)

        sudo('s3cmd setacl s3://%s/upload --acl-public --recursive --secret_key=%s --access_key=%s' % (
             dst_bucket_name, dst_bucket_secret_key, dst_bucket_access_key),
             user=env.remote_user)
        # cleanup
        sudo('rm -rf %s' % tmp_dir, warn_only=True, user=env.remote_user)
78901
def _get_s3_files(local_dir, file_info, params):
    
    assert len(file_info) == 1
    files = file_info.values()[0]
    fnames = []
    for k in ["1", "2"]:
        if files[k] not in fnames:
            fnames.append(files[k])
    out = []
    for fname in fnames:
        bucket, key = fname.replace("s3://", "").split("/", 1)
        if params["access_key_id"] == "TEST":
            out.append(os.path.join(local_dir, os.path.basename(key)))
        else:
            out.append(s3.get_file(local_dir, bucket, key, params))
    return out
582152
def download_run_and_upload(job, master_ip, inputs, spark_on_toil):
    
    master_ip = MasterAddress(master_ip)

    bam_name = inputs.sample.split('://')[-1].split('/')[-1]
    sample_name = ".".join(os.path.splitext(bam_name)[:-1])

    hdfs_subdir = sample_name + "-dir"

    if inputs.run_local:
        inputs.local_dir = job.fileStore.getLocalTempDir()
        if inputs.native_adam_path is None:
            hdfs_dir = "/data/"
        else:
            hdfs_dir = inputs.local_dir
    else:
        inputs.local_dir = None
        hdfs_dir = "hdfs://{0}:{1}/{2}".format(master_ip, HDFS_MASTER_PORT, hdfs_subdir)

    try:
        hdfs_prefix = hdfs_dir + "/" + sample_name
        hdfs_bam = hdfs_dir + "/" + bam_name

        hdfs_snps = hdfs_dir + "/" + inputs.dbsnp.split('://')[-1].split('/')[-1]

        if not inputs.run_local:
            download_data(job, master_ip, inputs, inputs.dbsnp, inputs.sample, hdfs_snps, hdfs_bam)
        else:
            copy_files([inputs.sample, inputs.dbsnp], inputs.local_dir)

        adam_input = hdfs_prefix + ".adam"
        adam_snps = hdfs_dir + "/snps.var.adam"
        adam_convert(job, master_ip, inputs, hdfs_bam, hdfs_snps, adam_input, adam_snps, spark_on_toil)

        adam_output = hdfs_prefix + ".processed.bam"
        adam_transform(job, master_ip, inputs, adam_input, adam_snps, hdfs_dir, adam_output, spark_on_toil)

        out_file = inputs.output_dir + "/" + sample_name + inputs.suffix + ".bam"

        if not inputs.run_local:
            upload_data(job, master_ip, inputs, adam_output, out_file, spark_on_toil)
        else:
            local_adam_output = "%s/%s.processed.bam" % (inputs.local_dir, sample_name)
            move_files([local_adam_output], inputs.output_dir)

        remove_file(master_ip, hdfs_subdir, spark_on_toil)
    except:
        remove_file(master_ip, hdfs_subdir, spark_on_toil)
        raise
288194
def update_file(finfo, sample_info, config):
    
    ffinal = filesystem.update_file(finfo, sample_info, config, pass_uptodate=True)
    if os.path.isdir(ffinal):
        to_transfer = []
        for path, dirs, files in os.walk(ffinal):
            for f in files:
                full_f = os.path.join(path, f)
                k = full_f.replace(os.path.abspath(config["dir"]) + "/", "")
                to_transfer.append((full_f, k))
    else:
        k = ffinal.replace(os.path.abspath(config["dir"]) + "/", "")
        to_transfer = [(ffinal, k)]

    region = "@%s" % config["region"] if config.get("region") else ""
    fname = "s3://%s%s/%s" % (config["bucket"], region, to_transfer[0][1])
    conn = objectstore.connect(fname)
    bucket = conn.lookup(config["bucket"])
    if not bucket:
        bucket = conn.create_bucket(config["bucket"], location=config.get("region", "us-east-1"))

    for fname, orig_keyname in to_transfer:
        keyname = os.path.join(config.get("folder", ""), orig_keyname)
        key = bucket.get_key(keyname) if bucket else None
        modified = datetime.datetime.fromtimestamp(email.utils.mktime_tz(
            email.utils.parsedate_tz(key.last_modified))) if key else None
        no_upload = key and modified >= finfo["mtime"]
        if not no_upload:
            _upload_file_aws_cli(fname, config["bucket"], keyname, config, finfo)
1045896
function writeToDisk(dataURL, fileName) {
    var fileExtension = fileName.split('.').pop(),
        fileRootNameWithBase = './uploads/' + fileName,
        filePath = fileRootNameWithBase,
        fileID = 2,
        fileBuffer;

    // @todo return the new filename to client
    while (fs.existsSync(filePath)) {
        filePath = fileRootNameWithBase + '(' + fileID + ').' + fileExtension;
        fileID += 1;
    }

    dataURL = dataURL.split(',').pop();
    fileBuffer = new Buffer(dataURL, 'base64');
    fs.writeFileSync(filePath, fileBuffer);

    console.log('filePath', filePath);
}
120318
def save_hdf(self,filename,path=''):
        
        self.orbpop_long.save_hdf(filename,'{}/long'.format(path))
        self.orbpop_short.save_hdf(filename,'{}/short'.format(path))
1122630
def resource_to_bucket_and_key(resource_name):
        
        if not resource_name.startswith(S3Meterer.s3_url_prefix):
            raise ValueError("Expected resource_name to start with %s" %
                             S3Meterer.s3_url_prefix)

        s3_bucket_and_key = resource_name[len(S3Meterer.s3_url_prefix):]

        try:
            bucket, key = s3_bucket_and_key.split("/", 1)
            if not bucket:
                raise ValueError("Bucket name cannot be empty in URL %r" %
                                 resource_name)

            return (bucket, key)
        except ValueError:
            raise ValueError("No key specified in URL %r" % resource_name)
377235
def upload_numpy_to_s3_shards(num_shards, s3, bucket, key_prefix, array, labels=None):
    """"""
    shards = _build_shards(num_shards, array)
    if labels is not None:
        label_shards = _build_shards(num_shards, labels)
    uploaded_files = []
    if key_prefix[-1] != '/':
        key_prefix = key_prefix + '/'
    try:
        for shard_index, shard in enumerate(shards):
            with tempfile.TemporaryFile() as file:
                if labels is not None:
                    write_numpy_to_dense_tensor(file, shard, label_shards[shard_index])
                else:
                    write_numpy_to_dense_tensor(file, shard)
                file.seek(0)
                shard_index_string = str(shard_index).zfill(len(str(len(shards))))
                file_name = "matrix_{}.pbr".format(shard_index_string)
                key = key_prefix + file_name
                logger.debug("Creating object {} in bucket {}".format(key, bucket))
                s3.Object(bucket, key).put(Body=file)
                uploaded_files.append(file_name)
        manifest_key = key_prefix + ".amazon.manifest"
        manifest_str = json.dumps(
            [{'prefix': 's3://{}/{}'.format(bucket, key_prefix)}] + uploaded_files)
        s3.Object(bucket, manifest_key).put(Body=manifest_str.encode('utf-8'))
        return "s3://{}/{}".format(bucket, manifest_key)
    except Exception as ex:  # pylint: disable=broad-except
        try:
            for file in uploaded_files:
                s3.Object(bucket, key_prefix + file).delete()
        finally:
            raise ex
760395
def upload(name):
    ''''''
    storage = fs.by_name(name)
    return jsonify(success=True, **handle_upload(storage))
395597
private function processFile($upload_path, $filename, $data, $id = null, $preserveFilePath = false) {
		if(!file_exists($upload_path.'/'.$filename)) {
            Logger::error("\tFile $filename is missing.");
			return $data;
		}

		if(isset($data['s3'])) {
			$key = $data['s3']['key'];

			if($key == $filename) {
				return $data;
			}

			$this->deleteFile($key);
		}

		$shouldUseCustomPrefix = (!empty(StorageSettings::prefixFormat()) && apply_filters('ilab_storage_should_use_custom_prefix', true));

		if (!$preserveFilePath && !isset($data['prefix']) && !$shouldUseCustomPrefix) {
		    $fpath = pathinfo($data['file'],PATHINFO_DIRNAME);
		    $fpath = str_replace($upload_path, '', $fpath);
		    $prefix = trailingslashit(ltrim($fpath, DIRECTORY_SEPARATOR));
            if ($prefix == './') {
                $prefix = trailingslashit(pathinfo($filename, PATHINFO_DIRNAME));
            }
        } else {
            $prefix = ($preserveFilePath && isset($data['prefix'])) ? $data['prefix'].DIRECTORY_SEPARATOR : StorageSettings::prefix($id);
        }

        $parts = explode('/', $filename);
        $bucketFilename = array_pop($parts);

		try {
            Logger::info("\tUploading $filename to S3.");
            $url = $this->client->upload($prefix.$bucketFilename, $upload_path.'/'.$filename, StorageSettings::privacy(), StorageSettings::cacheControl(), StorageSettings::expires());
            Logger::info("\tFinished uploading $filename to S3.");

			$options = [];
			$params = [];
			if(!empty(StorageSettings::cacheControl())) {
				$params['CacheControl'] = StorageSettings::cacheControl();
			}

			if(!empty(StorageSettings::expires())) {
				$params['Expires'] = StorageSettings::expires();
			}

			if(!empty($params)) {
				$options['params'] = $params;
			}

			$providerClass = get_class($this->client);
			$providerId = $providerClass::identifier();

			$data['s3'] = [
				'url' => $url,
				'bucket' => $this->client->bucket(),
				'privacy' => StorageSettings::privacy(),
				'key' => $prefix.$bucketFilename,
                'provider' =>  $providerId,
				'options' => $options
			];

			if(file_exists($upload_path.'/'.$filename)) {
				$ftype = wp_check_filetype($upload_path.'/'.$filename);
				if(!empty($ftype) && isset($ftype['type'])) {
					$data['s3']['mime-type'] = $ftype['type'];
				}
			}
		}
		catch(StorageException $ex) {
			Logger::error('Upload Error', [
				'exception' => $ex->getMessage(),
				'prefix' => $prefix,
				'bucketFilename' => $bucketFilename,
				'privacy' => StorageSettings::privacy()
			]);
		}

		if(StorageSettings::deleteOnUpload()) {
			if(file_exists($upload_path.'/'.$filename)) {
				unlink($upload_path.'/'.$filename);
			}
		}

		return $data;
	}
895133
def save_hdf(self, filename, path='', overwrite=False, append=False):
        

        if os.path.exists(filename):
            store = pd.HDFStore(filename)
            if path in store:
                store.close()
                if overwrite:
                    os.remove(filename)
                elif not append:
                    raise IOError('{} in {} exists.  Set either overwrite or append option.'.format(path,filename))
            else:
                store.close()

        df = self.to_df()
        df.to_hdf(filename, path+'/df')
        with pd.HDFStore(filename) as store:
            # store = pd.HDFStore(filename)
            attrs = store.get_storer(path+'/df').attrs
            attrs.spectroscopy = self.spectroscopy
            attrs.parallax = self.parallax
            attrs.N = self._N
            attrs.index = self._index
            store.close()
686439
def sync_s3(self):
        """"""
        bucket, key = self.open_s3()
        for directory in self.DIRECTORIES:
            for root, dirs, files in os.walk(directory):
                self.upload_s3((bucket, key, self.AWS_BUCKET_NAME, directory), root, files, dirs)
1115159
def open(cls, filename):
        """"""
        import boto
        file_info = cls.parse_remote(filename)
        connection = cls.connect(filename)
        try:
            s3_bucket = connection.get_bucket(file_info.bucket)
        except boto.exception.S3ResponseError as error:
            # if we don't have bucket permissions but folder permissions,
            # try without validation
            if error.status == 403:
                s3_bucket = connection.get_bucket(file_info.bucket,
                                                  validate=False)
            else:
                raise

        s3_key = s3_bucket.get_key(file_info.key)
        if s3_key is None:
            raise ValueError("Did not find S3 key: %s" % filename)
        return S3Handle(s3_key)
1371123
def upload_to_s3(self, source_path, bucket_name, disable_progress=False):
        

        """
        try:
            self.s3_client.head_bucket(Bucket=bucket_name)
        except botocore.exceptions.ClientError:
            # This is really stupid S3 quirk. Technically, us-east-1 one has no S3,
            # it's actually "US Standard", or something.
            # More here: https://github.com/boto/boto3/issues/125
            if self.aws_region == 'us-east-1':
                self.s3_client.create_bucket(
                    Bucket=bucket_name,
                )
            else:
                self.s3_client.create_bucket(
                    Bucket=bucket_name,
                    CreateBucketConfiguration={'LocationConstraint': self.aws_region},
                )

            if self.tags:
                tags = {
                    'TagSet': [{'Key': key, 'Value': self.tags[key]} for key in self.tags.keys()]
                }
                self.s3_client.put_bucket_tagging(Bucket=bucket_name, Tagging=tags)

        if not os.path.isfile(source_path) or os.stat(source_path).st_size == 0:
            print("Problem with source file {}".format(source_path))
            return False

        dest_path = os.path.split(source_path)[1]
        try:
            source_size = os.stat(source_path).st_size
            print("Uploading {0} ({1})..".format(dest_path, human_size(source_size)))
            progress = tqdm(total=float(os.path.getsize(source_path)), unit_scale=True, unit='B', disable=disable_progress)

            # Attempt to upload to S3 using the S3 meta client with the progress bar.
            # If we're unable to do that, try one more time using a session client,
            # which cannot use the progress bar.
            # Related: https://github.com/boto/boto3/issues/611
            try:
                self.s3_client.upload_file(
                    source_path, bucket_name, dest_path,
                    Callback=progress.update
                )
            except Exception as e:  # pragma: no cover
                self.s3_client.upload_file(source_path, bucket_name, dest_path)

            progress.close()
        except (KeyboardInterrupt, SystemExit):  # pragma: no cover
            raise
        except Exception as e:  # pragma: no cover
            print(e)
            return False
        return True
112482
public static void copyToHDFS(String localFile, String hdfsFile) throws IOException {

    String hadoopConfPath; 

    if (Environment.getProperty("hadoop.conf.path") == null)
      hadoopConfPath = "../../../conf";
    else
      hadoopConfPath = Environment.getProperty("hadoop.conf.path");

    // Read the configuration for the Hadoop environment
    Configuration hadoopConf = new Configuration();
    hadoopConf.addResource(new Path(hadoopConfPath + "/hadoop-default.xml"));
    hadoopConf.addResource(new Path(hadoopConfPath + "/hadoop-site.xml"));

    // System.out.println(hadoopConf.get("hadoop.tmp.dir"));
    // System.out.println(hadoopConf.get("fs.default.name"));
    FileSystem fs = FileSystem.get(hadoopConf);

    // HadoopDFS deals with Path
    Path inFile = new Path("file://" + localFile);
    Path outFile = new Path(hadoopConf.get("fs.default.name") + hdfsFile);

     // Read from and write to new file
    Environment.logInfo("Uploading to HDFS (file " + outFile + ") ...");
    fs.copyFromLocalFile(false, inFile, outFile);
  }