Creates a Hadoop FS {@link URI} given a user-specified configKey. If the given configKey does not have an authority,
a default one is used instead, provided by the default root path.
169689
private URI createFileSystemURI(URI configKey) throws URISyntaxException, IOException {
    // Validate the scheme
    String configKeyScheme = configKey.getScheme();
    if (!configKeyScheme.startsWith(getSchemePrefix())) {
      throw new IllegalArgumentException(
          String.format("Scheme for configKey \"%s\" must begin with \"%s\"!", configKey, getSchemePrefix()));
    }

    if (Strings.isNullOrEmpty(configKey.getAuthority())) {
      return new URI(getPhysicalScheme(), getDefaultStoreFsLazy().getUri().getAuthority(), "", "", "");
    }
    String uriPhysicalScheme = configKeyScheme.substring(getSchemePrefix().length(), configKeyScheme.length());
    return new URI(uriPhysicalScheme, configKey.getAuthority(), "", "", "");
  }
731922
public static List<URI> uris(AbstractConfig config, String key) {
    List<URI> result = new ArrayList<>();
    List<String> input = config.getList(key);
    for (String s : input) {
      result.add(uri(key, s));
    }
    return ImmutableList.copyOf(result);
  }
470998
public static FileSystem createInstance(
      Configuration config, Path workingDirectory, Collection<FileStatus> fileStatuses) {
    checkNotNull(config, "configuration can not be null");

    FileSystem fileSystem = new InMemoryGlobberFileSystem(workingDirectory, fileStatuses);
    fileSystem.setConf(config);

    return fileSystem;
  }
748500
public synchronized void initialize(URI uri, org.apache.hadoop.conf.Configuration conf,
      @Nullable AlluxioConfiguration alluxioConfiguration)
      throws IOException {
    Preconditions.checkArgument(uri.getScheme().equals(getScheme()),
        PreconditionMessage.URI_SCHEME_MISMATCH.toString(), uri.getScheme(), getScheme());

    super.initialize(uri, conf);
    LOG.debug("initialize({}, {}). Connecting to Alluxio", uri, conf);
    HadoopUtils.addSwiftCredentials(conf);
    setConf(conf);

    // HDFS doesn't allow the authority to be empty; it must be "/" instead.
    String authority = uri.getAuthority() == null ? "/" : uri.getAuthority();
    mAlluxioHeader = getScheme() + "://" + authority;
    // Set the statistics member. Use mStatistics instead of the parent class's variable.
    mStatistics = statistics;

    Authority auth = Authority.fromString(uri.getAuthority());
    if (auth instanceof UnknownAuthority) {
      throw new IOException(String.format("Authority \"%s\" is unknown. The client can not be "
              + "configured with the authority from %s", auth, uri));
    }

    mUri = URI.create(mAlluxioHeader);

    if (mFileSystem != null) {
      return;
    }

    Map<String, Object> uriConfProperties = getConfigurationFromUri(uri);

    AlluxioProperties alluxioProps =
        (alluxioConfiguration != null) ? alluxioConfiguration.copyProperties()
            : ConfigurationUtils.defaults();
    AlluxioConfiguration alluxioConf = mergeConfigurations(uriConfProperties, conf, alluxioProps);
    Subject subject = getHadoopSubject();
    if (subject != null) {
      LOG.debug("Using Hadoop subject: {}", subject);
    } else {
      LOG.debug("No Hadoop subject. Using context without subject.");
    }

    LOG.info("Initializing filesystem with connect details {}",
        Factory.getConnectDetails(alluxioConf));

    mFileSystem = FileSystem.Factory.create(ClientContext.create(subject, alluxioConf));
  }
48981
public static URI setPath(final URI initialUri, final String path) {
        String finalPath = path;
        if (!finalPath.startsWith("/")) {
            finalPath = '/' + path;
        }
        try {
            if (initialUri.getHost() == null && initialUri.getAuthority() != null) {
                return new URI(initialUri.getScheme(), initialUri.getAuthority(), finalPath,
                               initialUri.getQuery(),
                               initialUri.getFragment());
            } else {
                return new URI(initialUri.getScheme(), initialUri.getUserInfo(), initialUri.getHost(),
                               initialUri.getPort(),
                               finalPath, initialUri.getQuery(), initialUri.getFragment());
            }
        } catch (URISyntaxException e) {
            throw ExceptionUtils.getRuntimeException(e);
        }
    }
478817
@Override
	public void configure(Configuration parameters) {
		// get the file path
		String filePath = parameters.getString(FILE_PARAMETER_KEY, null);
		if (filePath != null) {
			try {
				this.filePath = new Path(filePath);
			}
			catch (RuntimeException rex) {
				throw new RuntimeException("Could not create a valid URI from the given file path name: " + rex.getMessage()); 
			}
		}
		else if (this.filePath == null) {
			throw new IllegalArgumentException("File path was not specified in input format, or configuration."); 
		}
	}
161122
function createVersion(config) {
  if (!(config && (typeof config.root === 'string'
      || config.root === null || config.root === undefined))) {
    throw new Error('"config.root" is required and must be a string');
  }
  var root = config.root;
  if (root === null || root === undefined) {
    root = defaultRoot;
  }

  return createRfr(function(idFromRoot) {
    if (typeof idFromRoot !== 'string') {
      throw new Error('A string is required for the argument of ' +
      'a user created RFR version.');
    }
    return require(normalizeId(idFromRoot, this.root));
  }, root);
}
557753
public static S3AUnderFileSystem createInstance(AlluxioURI uri, UnderFileSystemConfiguration conf,
      AlluxioConfiguration alluxioConf) {

    AWSCredentialsProvider credentials = createAwsCredentialsProvider(conf);
    String bucketName = UnderFileSystemUtils.getBucketName(uri);

    // Set the client configuration based on Alluxio configuration values.
    ClientConfiguration clientConf = new ClientConfiguration();

    // Socket timeout
    clientConf
        .setSocketTimeout((int) alluxioConf.getMs(PropertyKey.UNDERFS_S3A_SOCKET_TIMEOUT_MS));

    // HTTP protocol
    if (Boolean.parseBoolean(conf.get(PropertyKey.UNDERFS_S3A_SECURE_HTTP_ENABLED))) {
      clientConf.setProtocol(Protocol.HTTPS);
    } else {
      clientConf.setProtocol(Protocol.HTTP);
    }

    // Proxy host
    if (conf.isSet(PropertyKey.UNDERFS_S3_PROXY_HOST)) {
      clientConf.setProxyHost(conf.get(PropertyKey.UNDERFS_S3_PROXY_HOST));
    }

    // Proxy port
    if (conf.isSet(PropertyKey.UNDERFS_S3_PROXY_PORT)) {
      clientConf.setProxyPort(Integer.parseInt(conf.get(PropertyKey.UNDERFS_S3_PROXY_PORT)));
    }

    // Number of metadata and I/O threads to S3.
    int numAdminThreads = Integer.parseInt(conf.get(PropertyKey.UNDERFS_S3_ADMIN_THREADS_MAX));
    int numTransferThreads =
        Integer.parseInt(conf.get(PropertyKey.UNDERFS_S3_UPLOAD_THREADS_MAX));
    int numThreads = Integer.parseInt(conf.get(PropertyKey.UNDERFS_S3_THREADS_MAX));
    if (numThreads < numAdminThreads + numTransferThreads) {
      LOG.warn("Configured s3 max threads ({}) is less than # admin threads ({}) plus transfer "
          + "threads ({}). Using admin threads + transfer threads as max threads instead.",
              numThreads, numAdminThreads, numTransferThreads);
      numThreads = numAdminThreads + numTransferThreads;
    }
    clientConf.setMaxConnections(numThreads);

    // Set client request timeout for all requests since multipart copy is used,
    // and copy parts can only be set with the client configuration.
    clientConf
        .setRequestTimeout((int) alluxioConf.getMs(PropertyKey.UNDERFS_S3A_REQUEST_TIMEOUT));

    boolean streamingUploadEnabled =
        conf.getBoolean(PropertyKey.UNDERFS_S3A_STREAMING_UPLOAD_ENABLED);

    // Signer algorithm
    if (conf.isSet(PropertyKey.UNDERFS_S3A_SIGNER_ALGORITHM)) {
      clientConf.setSignerOverride(conf.get(PropertyKey.UNDERFS_S3A_SIGNER_ALGORITHM));
    }

    AmazonS3Client amazonS3Client = new AmazonS3Client(credentials, clientConf);

    // Set a custom endpoint.
    if (conf.isSet(PropertyKey.UNDERFS_S3_ENDPOINT)) {
      amazonS3Client.setEndpoint(conf.get(PropertyKey.UNDERFS_S3_ENDPOINT));
    }

    // Disable DNS style buckets, this enables path style requests.
    if (Boolean.parseBoolean(conf.get(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS))) {
      S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
      amazonS3Client.setS3ClientOptions(clientOptions);
    }

    ExecutorService service = ExecutorServiceFactories
        .fixedThreadPool("alluxio-s3-transfer-manager-worker",
            numTransferThreads).create();

    TransferManager transferManager = TransferManagerBuilder.standard()
        .withS3Client(amazonS3Client).withExecutorFactory(() -> service)
        .withMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD)
        .build();

    return new S3AUnderFileSystem(uri, amazonS3Client, bucketName,
        service, transferManager, conf, alluxioConf, streamingUploadEnabled);
  }
703365
public DfuServiceInitiator setZip(@NonNull final Uri uri) {
		return init(uri, null, 0, DfuBaseService.TYPE_AUTO, DfuBaseService.MIME_TYPE_ZIP);
	}
1133712
private void setConfigurationParameters(Configuration conf)
    throws IOException {
    if (conf.getBoolean("hadoop.disable.shell", false)) {
      conf.setStrings(UnixUserGroupInformation.UGI_PROPERTY_NAME, new String[]{"hadoop", "hadoop"});
      Shell.setDisabled(true);
    }

    try {
      fsOwner = UnixUserGroupInformation.login(conf);
    } catch (LoginException e) {
      throw new IOException(StringUtils.stringifyException(e));
    }

    LOG.info("fsOwner=" + fsOwner);

    this.hasRwLock = conf.getBoolean("dfs.rwlock", false);
    this.supergroup = conf.get("dfs.permissions.supergroup", "supergroup");
    this.isPermissionEnabled = conf.getBoolean("dfs.permissions", true);
    this.setPersistBlocks(conf.getBoolean("dfs.persist.blocks", false));
    LOG.info("supergroup=" + supergroup);
    LOG.info("isPermissionEnabled=" + isPermissionEnabled);
    short filePermission = (short) conf.getInt("dfs.upgrade.permission", 0777);
    this.defaultPermission = PermissionStatus.createImmutable(
      fsOwner.getUserName(), supergroup, new FsPermission(filePermission));


    this.maxCorruptFilesReturned = conf.getInt("dfs.corruptfilesreturned.max",
      DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED);
    this.defaultReplication = conf.getInt("dfs.replication", 3);
    this.maxReplication = conf.getInt("dfs.replication.max", 512);
    this.minReplication = conf.getInt("dfs.replication.min", 1);
    this.minCloseReplication = conf.getInt("dfs.close.replication.min",
        minReplication);
    if (minReplication <= 0) {
      throw new IOException(
        "Unexpected configuration parameters: dfs.replication.min = "
          + minReplication
          + " must be greater than 0");
    }
    if (maxReplication >= (int) Short.MAX_VALUE) {
      throw new IOException(
        "Unexpected configuration parameters: dfs.replication.max = "
          + maxReplication + " must be less than " + (Short.MAX_VALUE));
    }
    if (maxReplication < minReplication) {
      throw new IOException(
        "Unexpected configuration parameters: dfs.replication.min = "
          + minReplication
          + " must be less than dfs.replication.max = "
          + maxReplication);
    }
    if (minCloseReplication < minReplication) {
      throw new IOException(
          "Unexpected configuration parameters: dfs.close.replication.min = "
          + minCloseReplication
          + " must be no less than dfs.replication.min = "
          + minReplication);
    }
    if (minCloseReplication > maxReplication) {
      throw new IOException(
          "Unexpected configuration paramerters: dfs.close.replication.min = "
          + minCloseReplication
          + " must be no greater than dfs.replication.max = "
          + maxReplication);
    }
    ReplicationConfigKeys.updateConfigKeys(conf);
    this.maxReplicationStreams = conf.getInt("dfs.max-repl-streams", 2);
    long heartbeatInterval = conf.getLong("dfs.heartbeat.interval", 3) * 1000;
    long heartbeatRecheckInterval = conf.getInt(
      "heartbeat.recheck.interval", 5 * 60 * 1000); // 5 minutes
    setHeartbeatInterval(heartbeatInterval, heartbeatRecheckInterval);
    this.defaultBlockSize = conf.getLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
    this.maxFsObjects = conf.getLong("dfs.max.objects", 0);
    this.accessTimePrecision = conf.getLong("dfs.access.time.precision", 0);
    this.supportAppends = conf.getBoolean("dfs.support.append", false);
    this.accessTimeTouchable = conf.getBoolean("dfs.access.time.touchable", true);

    long editPreallocateSize = conf.getLong("dfs.edit.preallocate.size",
      HdfsConstants.DEFAULT_EDIT_PREALLOCATE_SIZE);
    FSEditLog.setPreallocateSize(editPreallocateSize);
    int editBufferSize = conf.getInt("dfs.edit.buffer.size",
      HdfsConstants.DEFAULT_EDIT_BUFFER_SIZE);
    FSEditLog.setBufferCapacity(editBufferSize);
    int maxBufferedTransactions = conf.getInt("dfs.max.buffered.transactions",
      HdfsConstants.DEFAULT_MAX_BUFFERED_TRANSACTIONS);
    FSEditLog.setMaxBufferedTransactions(maxBufferedTransactions);

    // Permission violations are logged in the namenode logs. The operation
    // does not fail.
    this.permissionAuditOnly = conf.getBoolean("dfs.permissions.audit.log", false);

    // set soft and hard lease period
    long hardLeaseLimit = conf.getLong(FSConstants.DFS_HARD_LEASE_KEY,
        FSConstants.LEASE_HARDLIMIT_PERIOD);
    long softLeaseLimit = conf.getLong(FSConstants.DFS_SOFT_LEASE_KEY,
        FSConstants.LEASE_SOFTLIMIT_PERIOD);
    this.leaseManager.setLeasePeriod(
        Math.min(hardLeaseLimit, softLeaseLimit), hardLeaseLimit);
  }
958187
protected URI createUri(String file,
      DatanodeID[] candidates,  UnixUserGroupInformation ugi,
      HttpServletRequest request) throws URISyntaxException {
    String scheme = request.getScheme();
    final DatanodeID host = candidates[0];
    final String hostname;
    if (host instanceof DatanodeInfo) {
      hostname = ((DatanodeInfo)host).getHostName();
    } else {
      hostname = host.getHost();
    }
    
    // Construct query.
    StringBuilder builder = new StringBuilder();
    builder.append("ugi=" + ugi);
    
    // Populate the rest of parameters.
    Enumeration<?> it = request.getParameterNames();
    while (it.hasMoreElements()) {
      String key = it.nextElement().toString();
      String value = request.getParameter(key);
      builder.append("&" + key + "=" + value);
    }

    // Construct the possible candidates for retry
    if (candidates.length > 1) {
      builder.append("&candidates=");
      appendDatanodeID(builder, candidates[1]);
      for (int j=2; j<candidates.length; j++) {
        builder.append(" ");
        appendDatanodeID(builder, candidates[j]);
      }
    }

    // Add namenode address to the url params
    NameNode nn = (NameNode)getServletContext().getAttribute("name.node");
    String addr = NetUtils.toIpPort(nn.getNameNodeAddress());
    builder.append(JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, addr));

    return new URI(scheme, null, hostname,
        "https".equals(scheme)
          ? (Integer)getServletContext().getAttribute("datanode.https.port")
          : host.getInfoPort(),
        "/streamFile" + file, builder.toString(), null);
  }
236371
public Configuration initializeConf(String[] keys,
      Configuration conf, FileSystem fs) throws IOException {
    Configuration newConf = new Configuration(conf);
    if (fs == null) {
      fs = FileSystem.get(conf);
    }
    String suffix = fs.getUri().getAuthority();
    for (String key: keys) {
      String value = conf.get(key + "." + suffix);
      if (value != null) {
        newConf.set(key, value);
      }
    }
    return newConf;
  }
782577
def createFromAPIKey(self, apikey, maybeWriteDefault=False):
        
        self._data = {
            'default_key': 'default',
            'keys': {
                'default': {
                    'key': apikey,
                    'desc': 'imported API key'
                }
            }
        }
        self._keyID = 'default'
        self._doDefaults()
        if maybeWriteDefault:
            path = os.path.expanduser(self.DEFAULT_CONFIG_FILE)
            self.write(path)
1161854
public org.apache.hadoop.conf.Configuration getOrLoadHadoopConfig() {
		org.apache.hadoop.conf.Configuration hadoopConfig = this.hadoopConfig;
		if (hadoopConfig == null) {
			if (flinkConfig != null) {
				hadoopConfig = mirrorCertainHadoopConfig(loadHadoopConfigFromFlink());
			}
			else {
				LOG.warn("Flink configuration is not set prior to loading this configuration."
					+ " Cannot forward configuration keys from Flink configuration.");
				hadoopConfig = new org.apache.hadoop.conf.Configuration();
			}
		}
		this.hadoopConfig = hadoopConfig;
		return hadoopConfig;
	}
820188
public static URI uri(AbstractConfig config, String key) {
    final String value = config.getString(key);
    return uri(key, value);
  }
1039147
public static GoogleCloudStorageItemInfo createInferredDirectory(StorageResourceId resourceId) {
    checkArgument(resourceId != null, "resourceId must not be null");
    return new GoogleCloudStorageItemInfo(
        resourceId,
        /* creationTime= */ 0,
        /* size= */ 0,
        /* location= */ null,
        /* storageClass= */ null);
  }
429795
public BucketingSink<T> setFSConfig(org.apache.hadoop.conf.Configuration config) {
		this.fsConfig = new Configuration();
		for (Map.Entry<String, String> entry : config) {
			fsConfig.setString(entry.getKey(), entry.getValue());
		}
		return this;
	}