Gets the Node list of input split locations sorted in rack order.
503244
public String getSplitNodes() {
    if (!isMapTask() || jobSetup || jobCleanup) {
      return "";
    }
    String[] nodes = rawSplit.getLocations();
    if (nodes == null || nodes.length == 0) {
      return "";
    }
    StringBuffer ret = new StringBuffer(nodes[0]);
    for(int i = 1; i < nodes.length;i++) {
      ret.append(",");
      ret.append(nodes[i]);
    }
    return ret.toString();
  }
1297111
private static List<InputSplit> cleanSplits(List<InputSplit> splits) throws IOException {
    if (VersionInfo.getVersion().compareTo("2.3.0") >= 0) {
      // This issue was fixed in 2.3.0, if newer version, no need to clean up splits
      return splits;
    }

    List<InputSplit> cleanedSplits = Lists.newArrayList();

    for (int i = 0; i < splits.size(); i++) {
      CombineFileSplit oldSplit = (CombineFileSplit) splits.get(i);
      String[] locations = oldSplit.getLocations();

      Preconditions.checkNotNull(locations, "CombineFileSplit.getLocations() returned null");

      if (locations.length > SPLIT_MAX_NUM_LOCATIONS) {
        locations = Arrays.copyOf(locations, SPLIT_MAX_NUM_LOCATIONS);
      }

      cleanedSplits.add(new CombineFileSplit(oldSplit.getPaths(), oldSplit.getStartOffsets(), oldSplit.getLengths(),
          locations));
    }
    return cleanedSplits;
  }
1238620
@Override public List<InputSplit> getSplits(JobContext job)
		throws IOException
	{
		final List<InputSplit> splits = super.getSplits(job);

		// Align the splits so that they don't cross blocks

		// addIndexedSplits() requires the given splits to be sorted by file
		// path, so do so. Although FileInputFormat.getSplits() does, at the time
		// of writing this, generate them in that order, we shouldn't rely on it.
		Collections.sort(splits, new Comparator<InputSplit>() {
			public int compare(InputSplit a, InputSplit b) {
				FileSplit fa = (FileSplit)a, fb = (FileSplit)b;
				return fa.getPath().compareTo(fb.getPath());
			}
		});

		final List<InputSplit> newSplits =
			new ArrayList<InputSplit>(splits.size());

		final Configuration cfg = job.getConfiguration();

		for (int i = 0; i < splits.size();) {
			try {
				i = addIndexedSplits      (splits, i, newSplits, cfg);
			} catch (IOException e) {
				i = addProbabilisticSplits(splits, i, newSplits, cfg);
			}
		}
		return newSplits;
	}
1128142
public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
    setFormat(job);
    job.setLong("mapred.min.split.size", Long.MAX_VALUE);
    return root.getSplits(job, numSplits);
  }
599760
def _get_sorted(self, resources):
        
        tmp = []
        for resource in resources:
            path = resource._path
            # Each slash counts as 10 priority, each variable takes one away
            priority = path.count('/') * 10 - path.count('{')
            tmp.append((priority, resource))
        return [resource for prio, resource in reversed(sorted(tmp))]
936348
def all_locations(self):
        '''
        
        '''
        qs = self.assoc_locations.all()
        for node in self.get_descendants():
            qs2 = node.assoc_locations.all()
            qs = qs.union(qs2).distinct('pk')
        return qs
1247767
public Iterable<Entry<String, ProfileNode>> getDescendingOrderedNodes() {
			if( nodes == null ) return null;
			
			List<Entry<String, ProfileNode>> averageStepTimes = new LinkedList<Entry<String, ProfileNode>>(nodes.entrySet());
			
			Collections.sort(averageStepTimes, new Comparator<Entry<String, ProfileNode>>() {
				public int compare(Entry<String, ProfileNode> o1, Entry<String, ProfileNode> o2) {
					return (int) (o1.getValue().totalMillisecs - o2.getValue().totalMillisecs);
				}
			});
			
			
			return averageStepTimes;
	}
487971
def get_resps(self, source_tree):
        
        resps = set()
        bearers = source_tree.xpath('//*[@resp]',
                                    namespaces=constants.NAMESPACES)
        for bearer in bearers:
            for resp in resp_splitter.split(bearer.get('resp')):
                if resp:
                    resps.add(tuple(resp.split('|', maxsplit=1)))
        return sorted(resps), bearers
894694
def list_nodes(full=False, call=None):
    '''
    
    '''
    if call == 'action':
        raise SaltCloudSystemExit(
            'The list_nodes function must be called with -f or --function.'
        )

    ret = {}
    if POLL_ALL_LOCATIONS:
        for location in JOYENT_LOCATIONS:
            result = query(command='my/machines', location=location,
                           method='GET')
            if result[0] in VALID_RESPONSE_CODES:
                nodes = result[1]
                for node in nodes:
                    if 'name' in node:
                        node['location'] = location
                        ret[node['name']] = reformat_node(item=node, full=full)
            else:
                log.error('Invalid response when listing Joyent nodes: %s', result[1])

    else:
        location = get_location()
        result = query(command='my/machines', location=location,
                       method='GET')
        nodes = result[1]
        for node in nodes:
            if 'name' in node:
                node['location'] = location
                ret[node['name']] = reformat_node(item=node, full=full)
    return ret
861782
def split_input(cls, mapper_spec):
    """"""
    filelists = mapper_spec.params[cls.FILES_PARAM]
    max_values_count = mapper_spec.params.get(cls.MAX_VALUES_COUNT_PARAM, -1)
    max_values_size = mapper_spec.params.get(cls.MAX_VALUES_SIZE_PARAM, -1)
    return [cls([0] * len(files), max_values_count, max_values_size)
            for files in filelists]
70336
public Collection<String> getLocalPlacements() {
        List<String> placements = Lists.newArrayList();
        for (String placementName : _placementFactory.getValidPlacements()) {
            Placement placement;
            try {
                placement = get(placementName);
            } catch (UnknownPlacementException e) {
                continue;  // Placement must live in another data center.
            }
            placements.add(placement.getName());
        }
        return placements;
    }
555511
@Override
  public NumberedSplit<InputSplit> getInputSplit(final NodeDescriptor nodeDescriptor, final String evaluatorId) {
    synchronized (evaluatorToSplits) {
      if (evaluatorToSplits.containsKey(evaluatorId)) {
        LOG.log(Level.FINE, "Found an already allocated split, {0}", evaluatorToSplits.toString());
        return evaluatorToSplits.get(evaluatorId);
      }
    }
    // always first try to allocate based on the hostName
    final String hostName = nodeDescriptor.getName();
    LOG.log(Level.FINE, "Allocated split not found, trying on {0}", hostName);
    if (locationToSplits.containsKey(hostName)) {
      LOG.log(Level.FINE, "Found splits possibly hosted for {0} at {1}", new Object[] {evaluatorId, hostName});
      final NumberedSplit<InputSplit> split = allocateSplit(evaluatorId, locationToSplits.get(hostName));
      if (split != null) {
        return split;
      }
    }
    LOG.log(Level.FINE, "{0} does not host any splits or someone else took splits hosted here. Picking other ones",
        hostName);
    final NumberedSplit<InputSplit> split = tryAllocate(nodeDescriptor, evaluatorId);
    if (split == null) {
      throw new RuntimeException("Unable to find an input split to evaluator " + evaluatorId);
    } else {
      LOG.log(Level.FINE, evaluatorToSplits.toString());
    }
    return split;
  }
1108084
def sorted(self):
        
        if self.start < self.end:
            return self.start, self.end
        else:
            return self.end, self.start
660848
def _find_read_splits(in_file, split_size):
    
    num_lines = total_reads_from_grabix(in_file) * 4
    assert num_lines and num_lines > 0, "Did not find grabix index reads: %s %s" % (in_file, num_lines)
    split_lines = split_size * 4
    chunks = []
    last = 1
    for chunki in range(num_lines // split_lines + min(1, num_lines % split_lines)):
        new = last + split_lines - 1
        chunks.append((last, min(new, num_lines)))
        last = new + 1
    return ["%s-%s" % (s, e) for s, e in chunks]
9677
def _getSynIngestNodes(self, item):
        '''
        
        '''
        pnodes = []
        seen = item.get('seen')
        # Track all the ndefs we make so we can make sources
        ndefs = []

        # Make the form nodes
        tags = item.get('tags', {})
        forms = item.get('forms', {})
        for form, valus in forms.items():
            for valu in valus:
                ndef = [form, valu]
                ndefs.append(ndef)
                obj = [ndef, {'tags': tags}]
                if seen:
                    obj[1]['props'] = {'.seen': seen}
                pnodes.append(obj)

        # Make the packed nodes
        nodes = item.get('nodes', ())
        for pnode in nodes:
            ndefs.append(pnode[0])
            pnode[1].setdefault('tags', {})
            for tag, valu in tags.items():
                # Tag in the packed node has a higher predecence
                # than the tag in the whole ingest set of data.
                pnode[1]['tags'].setdefault(tag, valu)
            if seen:
                pnode[1].setdefault('props', {})
                pnode[1]['props'].setdefault('.seen', seen)
            pnodes.append(pnode)

        # Make edges
        for srcdef, etyp, destndefs in item.get('edges', ()):
            for destndef in destndefs:
                ndef = [etyp, [srcdef, destndef]]
                ndefs.append(ndef)
                obj = [ndef, {}]
                if seen:
                    obj[1]['props'] = {'.seen': seen}
                if tags:
                    obj[1]['tags'] = tags.copy()
                pnodes.append(obj)

        # Make time based edges
        for srcdef, etyp, destndefs in item.get('time:edges', ()):
            for destndef, time in destndefs:
                ndef = [etyp, [srcdef, destndef, time]]
                ndefs.append(ndef)
                obj = [ndef, {}]
                if seen:
                    obj[1]['props'] = {'.seen': seen}
                if tags:
                    obj[1]['tags'] = tags.copy()
                pnodes.append(obj)

        # Make the source node and links
        source = item.get('source')
        if source:
            # Base object
            obj = [['meta:source', source], {}]
            pnodes.append(obj)

            # Subsequent links
            for ndef in ndefs:
                obj = [['meta:seen', (source, ndef)],
                       {'props': {'.seen': seen}}]
                pnodes.append(obj)
        return pnodes
157020
private int addIndexedSplits(
			List<InputSplit> splits, int i, List<InputSplit> newSplits,
			Configuration cfg)
		throws IOException
	{
		final Path file = ((FileSplit)splits.get(i)).getPath();

		final BGZFBlockIndex idx = new BGZFBlockIndex(
			file.getFileSystem(cfg).open(getIdxPath(file)));

		int splitsEnd = splits.size();
		for (int j = i; j < splitsEnd; ++j)
			if (!file.equals(((FileSplit)splits.get(j)).getPath()))
				splitsEnd = j;

		for (int j = i; j < splitsEnd; ++j) {
			final FileSplit fileSplit = (FileSplit)splits.get(j);

			final long start =         fileSplit.getStart();
			final long end   = start + fileSplit.getLength();

			final Long blockStart = idx.prevBlock(start);
			final Long blockEnd   = j == splitsEnd-1 ? idx.prevBlock(end)
			                                         : idx.nextBlock(end);

			if (blockStart == null)
				throw new RuntimeException(
					"Internal error or invalid index: no block start for " +start);

			if (blockEnd == null)
				throw new RuntimeException(
					"Internal error or invalid index: no block end for " +end);

			newSplits.add(new FileSplit(
				file, blockStart, blockEnd - blockStart,
				fileSplit.getLocations()));
		}
		return splitsEnd;
	}
996422
def _get_sorted_inputs(filename):
  
  with tf.gfile.Open(filename) as f:
    records = f.read().split("\n")
    inputs = [record.strip() for record in records]
    if not inputs[-1]:
      inputs.pop()

  input_lens = [(i, len(line.split())) for i, line in enumerate(inputs)]
  sorted_input_lens = sorted(input_lens, key=lambda x: x[1], reverse=True)

  sorted_inputs = []
  sorted_keys = {}
  for i, (index, _) in enumerate(sorted_input_lens):
    sorted_inputs.append(inputs[index])
    sorted_keys[index] = i
  return sorted_inputs, sorted_keys