Partition and group on just the last name; sort on both last and first name.

@param conf the Hadoop config
779946
public static void setupSecondarySort(Configuration conf) {
        ShuffleUtils.configBuilder()
                .useNewApi()
                .setPartitionerIndices(TupleFields.LAST_NAME)
                .setSortIndices(TupleFields.values())
                .setGroupIndices(TupleFields.LAST_NAME)
                .configure(conf);
    }
1019539
public void initialize(URI name, Configuration conf) throws IOException {
    fs.initialize(name, conf);
  }
935673
private static Map<String, Object> processConf(Map<String, ?> conf) {
        Map<String, Object> m = new HashMap<String, Object>(conf.size());
        for (String s : conf.keySet()) {
            Object o = conf.get(s);
            if (s.startsWith("act.")) s = s.substring(4);
            m.put(s, o);
            m.put(Config.canonical(s), o);
        }
        return m;
    }
618846
public static org.apache.hadoop.conf.Configuration getHBaseConfiguration(Configuration atlasConf) throws AtlasException {
        Configuration subsetAtlasConf =
                ApplicationProperties.getSubsetConfiguration(atlasConf, CONFIG_PREFIX);
        org.apache.hadoop.conf.Configuration hbaseConf = HBaseConfiguration.create();
        Iterator<String> keys = subsetAtlasConf.getKeys();
        while (keys.hasNext()) {
            String key = keys.next();
            hbaseConf.set(key, subsetAtlasConf.getString(key));
        }
        return hbaseConf;
    }
269692
public static void initializeComparators(Configuration conf, TupleMRConfig groupConfig) {
    TupleMRConfigBuilder.initializeComparators(conf, groupConfig.getCommonCriteria());
    for (Criteria criteria : groupConfig.getSpecificOrderBys()) {
      if (criteria != null) {
        TupleMRConfigBuilder.initializeComparators(conf, criteria);
      }
    }
  }
1091867
public StreamingStep withHadoopConfig(String key, String value) {
    hadoopConfig.put(key, value);
    return this;
  }
47
@Override
	public void configure(Configuration parameters) {

		// enforce sequential configuration() calls
		synchronized (CONFIGURE_MUTEX) {
			if (mapreduceInputFormat instanceof Configurable) {
				((Configurable) mapreduceInputFormat).setConf(configuration);
			}
		}
	}
1330764
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
		int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
		conf.setInteger(CACHE_FILE_NUM, num);
		conf.setString(CACHE_FILE_NAME + num, name);
		conf.setString(CACHE_FILE_PATH + num, e.filePath);
		conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
		conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
		if (e.blobKey != null) {
			conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
		}
	}
1162866
public static void addToHadoopConfiguration(Configuration conf) {
    final String SERIALIZATION_KEY = "io.serializations";

    String existingSerializers = conf.get(SERIALIZATION_KEY);
    if (existingSerializers != null) {
      conf.set(SERIALIZATION_KEY, existingSerializers + "," + WritableShimSerialization.class.getName());
    } else {
      conf.set(SERIALIZATION_KEY,
          "org.apache.hadoop.io.serializer.WritableSerialization," + WritableShimSerialization.class.getName());
    }
  }
286057
public static String getUserNameInConf(Configuration jobConf)
			throws IllegalArgumentException {
		String userName = jobConf.get(Constants.USER_CONF_KEY_HADOOP2);
		if (StringUtils.isBlank(userName)) {
			userName = jobConf.get(Constants.USER_CONF_KEY);
			if (StringUtils.isBlank(userName)) {
				// neither user.name nor hadoop.mapreduce.job.user.name found
				throw new IllegalArgumentException(" Found neither "
						+ Constants.USER_CONF_KEY + " nor "
						+ Constants.USER_CONF_KEY_HADOOP2);
			}
		}
		return userName;
	}
399479
private static BucketConfiguration createDefaultConfig(String hostname, int numNodes, int bucketStartPort, int numVBuckets, int numReplicas) {
        BucketConfiguration defaultConfig = new BucketConfiguration();
        defaultConfig.type = BucketType.COUCHBASE;
        defaultConfig.hostname = hostname;
        defaultConfig.numNodes = numNodes;
        if (numReplicas > -1) {
            defaultConfig.numReplicas = numReplicas;
        }

        defaultConfig.bucketStartPort = bucketStartPort;
        defaultConfig.numVBuckets = numVBuckets;
        return defaultConfig;
    }
1071408
protected void setStagingFileGroup()
      throws IOException {
    Preconditions.checkArgument(this.fs.exists(this.stagingFile),
        String.format("Staging output file %s does not exist", this.stagingFile));
    if (this.group.isPresent()) {
      HadoopUtils.setGroup(this.fs, this.stagingFile, this.group.get());
    }
  }
1155777
public static org.apache.hadoop.conf.Configuration getHadoopConfiguration() {
		Configuration retConf = new org.apache.hadoop.conf.Configuration();

		// We need to load both core-site.xml and hdfs-site.xml to determine the default fs path and
		// the hdfs configuration
		// Try to load HDFS configuration from Hadoop's own configuration files
		// 1. approach: Stratosphere configuration
		final String hdfsDefaultPath = GlobalConfiguration.getString(ConfigConstants.HDFS_DEFAULT_CONFIG, null);
		if (hdfsDefaultPath != null) {
			retConf.addResource(new org.apache.hadoop.fs.Path(hdfsDefaultPath));
		} else {
			LOG.debug("Cannot find hdfs-default configuration file");
		}

		final String hdfsSitePath = GlobalConfiguration.getString(ConfigConstants.HDFS_SITE_CONFIG, null);
		if (hdfsSitePath != null) {
			retConf.addResource(new org.apache.hadoop.fs.Path(hdfsSitePath));
		} else {
			LOG.debug("Cannot find hdfs-site configuration file");
		}
		
		// 2. Approach environment variables
		String[] possibleHadoopConfPaths = new String[4]; 
		possibleHadoopConfPaths[0] = GlobalConfiguration.getString(ConfigConstants.PATH_HADOOP_CONFIG, null);
		possibleHadoopConfPaths[1] = System.getenv("HADOOP_CONF_DIR");
		
		if (System.getenv("HADOOP_HOME") != null) {
			possibleHadoopConfPaths[2] = System.getenv("HADOOP_HOME")+"/conf";
			possibleHadoopConfPaths[3] = System.getenv("HADOOP_HOME")+"/etc/hadoop"; // hadoop 2.2
		}
		
		for (int i = 0; i < possibleHadoopConfPaths.length; i++) {
			if (possibleHadoopConfPaths[i] == null) {
				continue;
			}
			
			if (new File(possibleHadoopConfPaths[i]).exists()) {
				if (new File(possibleHadoopConfPaths[i]+"/core-site.xml").exists()) {
					retConf.addResource(new org.apache.hadoop.fs.Path(possibleHadoopConfPaths[i]+"/core-site.xml"));
					
					if (LOG.isDebugEnabled()) {
						LOG.debug("Adding "+possibleHadoopConfPaths[i]+"/core-site.xml to hadoop configuration");
					}
				}
				if (new File(possibleHadoopConfPaths[i]+"/hdfs-site.xml").exists()) {
					retConf.addResource(new org.apache.hadoop.fs.Path(possibleHadoopConfPaths[i]+"/hdfs-site.xml"));
					
					if (LOG.isDebugEnabled()) {
						LOG.debug("Adding "+possibleHadoopConfPaths[i]+"/hdfs-site.xml to hadoop configuration");
					}
				}
			}
		}
		return retConf;
	}
286881
private void removeBucketConfig(final String name) {
        LOGGER.debug("Removing bucket {} configuration from known configs.", name);
        ClusterConfig cluster = currentConfig;
        cluster.deleteBucketConfig(name);
        currentConfig = cluster;
        configObservable.onNext(currentConfig);
    }
1133712
private void setConfigurationParameters(Configuration conf)
    throws IOException {
    if (conf.getBoolean("hadoop.disable.shell", false)) {
      conf.setStrings(UnixUserGroupInformation.UGI_PROPERTY_NAME, new String[]{"hadoop", "hadoop"});
      Shell.setDisabled(true);
    }

    try {
      fsOwner = UnixUserGroupInformation.login(conf);
    } catch (LoginException e) {
      throw new IOException(StringUtils.stringifyException(e));
    }

    LOG.info("fsOwner=" + fsOwner);

    this.hasRwLock = conf.getBoolean("dfs.rwlock", false);
    this.supergroup = conf.get("dfs.permissions.supergroup", "supergroup");
    this.isPermissionEnabled = conf.getBoolean("dfs.permissions", true);
    this.setPersistBlocks(conf.getBoolean("dfs.persist.blocks", false));
    LOG.info("supergroup=" + supergroup);
    LOG.info("isPermissionEnabled=" + isPermissionEnabled);
    short filePermission = (short) conf.getInt("dfs.upgrade.permission", 0777);
    this.defaultPermission = PermissionStatus.createImmutable(
      fsOwner.getUserName(), supergroup, new FsPermission(filePermission));


    this.maxCorruptFilesReturned = conf.getInt("dfs.corruptfilesreturned.max",
      DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED);
    this.defaultReplication = conf.getInt("dfs.replication", 3);
    this.maxReplication = conf.getInt("dfs.replication.max", 512);
    this.minReplication = conf.getInt("dfs.replication.min", 1);
    this.minCloseReplication = conf.getInt("dfs.close.replication.min",
        minReplication);
    if (minReplication <= 0) {
      throw new IOException(
        "Unexpected configuration parameters: dfs.replication.min = "
          + minReplication
          + " must be greater than 0");
    }
    if (maxReplication >= (int) Short.MAX_VALUE) {
      throw new IOException(
        "Unexpected configuration parameters: dfs.replication.max = "
          + maxReplication + " must be less than " + (Short.MAX_VALUE));
    }
    if (maxReplication < minReplication) {
      throw new IOException(
        "Unexpected configuration parameters: dfs.replication.min = "
          + minReplication
          + " must be less than dfs.replication.max = "
          + maxReplication);
    }
    if (minCloseReplication < minReplication) {
      throw new IOException(
          "Unexpected configuration parameters: dfs.close.replication.min = "
          + minCloseReplication
          + " must be no less than dfs.replication.min = "
          + minReplication);
    }
    if (minCloseReplication > maxReplication) {
      throw new IOException(
          "Unexpected configuration paramerters: dfs.close.replication.min = "
          + minCloseReplication
          + " must be no greater than dfs.replication.max = "
          + maxReplication);
    }
    ReplicationConfigKeys.updateConfigKeys(conf);
    this.maxReplicationStreams = conf.getInt("dfs.max-repl-streams", 2);
    long heartbeatInterval = conf.getLong("dfs.heartbeat.interval", 3) * 1000;
    long heartbeatRecheckInterval = conf.getInt(
      "heartbeat.recheck.interval", 5 * 60 * 1000); // 5 minutes
    setHeartbeatInterval(heartbeatInterval, heartbeatRecheckInterval);
    this.defaultBlockSize = conf.getLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
    this.maxFsObjects = conf.getLong("dfs.max.objects", 0);
    this.accessTimePrecision = conf.getLong("dfs.access.time.precision", 0);
    this.supportAppends = conf.getBoolean("dfs.support.append", false);
    this.accessTimeTouchable = conf.getBoolean("dfs.access.time.touchable", true);

    long editPreallocateSize = conf.getLong("dfs.edit.preallocate.size",
      HdfsConstants.DEFAULT_EDIT_PREALLOCATE_SIZE);
    FSEditLog.setPreallocateSize(editPreallocateSize);
    int editBufferSize = conf.getInt("dfs.edit.buffer.size",
      HdfsConstants.DEFAULT_EDIT_BUFFER_SIZE);
    FSEditLog.setBufferCapacity(editBufferSize);
    int maxBufferedTransactions = conf.getInt("dfs.max.buffered.transactions",
      HdfsConstants.DEFAULT_MAX_BUFFERED_TRANSACTIONS);
    FSEditLog.setMaxBufferedTransactions(maxBufferedTransactions);

    // Permission violations are logged in the namenode logs. The operation
    // does not fail.
    this.permissionAuditOnly = conf.getBoolean("dfs.permissions.audit.log", false);

    // set soft and hard lease period
    long hardLeaseLimit = conf.getLong(FSConstants.DFS_HARD_LEASE_KEY,
        FSConstants.LEASE_HARDLIMIT_PERIOD);
    long softLeaseLimit = conf.getLong(FSConstants.DFS_SOFT_LEASE_KEY,
        FSConstants.LEASE_SOFTLIMIT_PERIOD);
    this.leaseManager.setLeasePeriod(
        Math.min(hardLeaseLimit, softLeaseLimit), hardLeaseLimit);
  }
112482
public static void copyToHDFS(String localFile, String hdfsFile) throws IOException {

    String hadoopConfPath; 

    if (Environment.getProperty("hadoop.conf.path") == null)
      hadoopConfPath = "../../../conf";
    else
      hadoopConfPath = Environment.getProperty("hadoop.conf.path");

    // Read the configuration for the Hadoop environment
    Configuration hadoopConf = new Configuration();
    hadoopConf.addResource(new Path(hadoopConfPath + "/hadoop-default.xml"));
    hadoopConf.addResource(new Path(hadoopConfPath + "/hadoop-site.xml"));

    // System.out.println(hadoopConf.get("hadoop.tmp.dir"));
    // System.out.println(hadoopConf.get("fs.default.name"));
    FileSystem fs = FileSystem.get(hadoopConf);

    // HadoopDFS deals with Path
    Path inFile = new Path("file://" + localFile);
    Path outFile = new Path(hadoopConf.get("fs.default.name") + hdfsFile);

     // Read from and write to new file
    Environment.logInfo("Uploading to HDFS (file " + outFile + ") ...");
    fs.copyFromLocalFile(false, inFile, outFile);
  }
778577
public void setSortName(String sortName) {

        if (sortName.equals(SORT_NAMES[1])) {
            m_sort = SORT_DATE_CREATED;
        } else if (sortName.equals(SORT_NAMES[2])) {
            m_sort = SORT_DATE_LASTMODIFIED;
        } else if (sortName.equals(SORT_NAMES[3])) {
            m_sort = SORT_TITLE;
        } else {
            m_sort = SORT_DEFAULT;
        }
    }