Create a division of the input files in chunks.

    The result is stored to a JSON file.
341376
def create_chunked_list(in_dir, size, out_dir, out_name):
    
    create_dirs(out_dir)

    in_files = get_files(in_dir)
    chunks = chunk(in_files, size)

    division = {}

    for i, files in enumerate(chunks):
        division[i] = [os.path.basename(f) for f in files]

    out_file = os.path.join(out_dir, out_name)
    with codecs.open(out_file, 'wb', encoding='utf-8') as f:
        json.dump(division, f, indent=4)
421401
def create_asset(json):
        
        result = Asset(json['sys'])
        file_dict = json['fields']['file']
        result.fields = json['fields']
        result.url = file_dict['url']
        result.mimeType = file_dict['contentType']
        return result
307323
def jsonify(result, format=False):
    '''  '''

    result2 = result.copy()
    if format:
        return json.dumps(result2, sort_keys=True, indent=4)
    else:
        return json.dumps(result2, sort_keys=True)
348424
def ParseMultiple(self, stats, file_objs, kb):
    """"""
    fileset = {stat.pathspec.path: obj for stat, obj in zip(stats, file_objs)}
    return self.ParseFileset(fileset)
761609
def split_input(self, input_filepath):
        """"""
        with open(input_filepath, 'r') as input_file:
            input_file_str = input_file.read()
            merge_file_str, parsetree_str = input_file_str.split('ParentedTree', 1)
            return merge_file_str, 'ParentedTree' + parsetree_str
1187954
def _file_chunks(self, data, chunk_size):
        
        for i in xrange(0, len(data), chunk_size):
            yield self.compressor(data[i:i+chunk_size])
93358
def splitter(iterable, chunksize=60):
    """"""
    return (iterable[0+i:chunksize+i]
            for i in range(0, len(iterable), chunksize))
568165
@SuppressWarnings("deprecation")
  public InputSplit[] getSplits(JobConf job, int numSplits)
    throws IOException {
    LocatedFileStatus[] files = listLocatedStatus(job);
    
    long totalSize = 0;                           // compute total size
    for (FileStatus file: files) {                // check we have valid files
      if (file.isDir()) {
        throw new IOException("Not a file: "+ file.getPath());
      }
      totalSize += file.getLen();
    }

    long goalSize = totalSize / (numSplits == 0 ? 1 : numSplits);
    long minSize = Math.max(job.getLong("mapred.min.split.size", 1),
                            minSplitSize);

    // generate splits
    ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
    NetworkTopology clusterMap = new NetworkTopology();
    for (LocatedFileStatus file: files) {
      Path path = file.getPath();
      FileSystem fs = path.getFileSystem(job);
      long length = file.getLen();
      BlockLocation[] blkLocations = file.getBlockLocations();

      if ((length != 0) && isSplitable(fs, path)) { 
        long blockSize = file.getBlockSize();
        long splitSize = computeSplitSize(goalSize, minSize, blockSize);

        long bytesRemaining = length;
        while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
          String[] splitHosts = getSplitHosts(blkLocations, 
              length-bytesRemaining, splitSize, clusterMap);
          splits.add(new FileSplit(path, length-bytesRemaining, splitSize, 
              splitHosts));
          bytesRemaining -= splitSize;
        }
        
        if (bytesRemaining != 0) {
          splits.add(new FileSplit(path, length-bytesRemaining, bytesRemaining, 
                     blkLocations[blkLocations.length-1].getHosts()));
        }
      } else if (length != 0) {
        String[] splitHosts = getSplitHosts(blkLocations,0,length,clusterMap);
        splits.add(new FileSplit(path, 0, length, splitHosts));
      } else { 
        //Create empty hosts array for zero length files
        splits.add(new FileSplit(path, 0, length, new String[0]));
      }
    }
    LOG.debug("Total # of splits: " + splits.size());
    return splits.toArray(new FileSplit[splits.size()]);
  }
108385
@GwtIncompatible("Unnecessary")
  private List<SourceFile> createInputs(
      List<FlagEntry<JsSourceType>> files,
      List<JsonFileSpec> jsonFiles,
      List<JsModuleSpec> jsModuleSpecs)
      throws IOException {
    return createInputs(files, jsonFiles, /* allowStdIn= */ false, jsModuleSpecs);
  }
56515
public static void splitFile(File file, long[] splitPoints, String storageFolder) throws IOException {
        splitPoints = ArrayUtils.concatArrays(new long[]{0}, splitPoints, new long[]{file.length()});
        String[] fileName = file.getName().split("\\.", 2);
        storageFolder += ValueConsts.SEPARATOR;
        logger.info("start to split file '" + file.getAbsolutePath() + "'");
        for (int i = 0; i < splitPoints.length - 1; i++) {
            long start = splitPoints[i], end = splitPoints[i + 1];
            long length = end - start;
            String filePath = storageFolder + fileName[0] + "_" + (i + 1) + "." + fileName[1];
            createNewFile(filePath);
            while (length > Integer.MAX_VALUE) {
                saveFile(filePath, readFile(file, start, Integer.MAX_VALUE), true);
                start += Integer.MAX_VALUE;
                length -= Integer.MAX_VALUE;
            }
            if (length > 0) {
                saveFile(filePath, readFile(file, start, (int) length), true);
            }
        }
    }
1185741
def chunks(self, size=32, alignment=1):
        

        if (size % alignment) != 0:
            raise Error(
                'size {} is not a multiple of alignment {}'.format(
                    size,
                    alignment))

        for segment in self:
            for chunk in segment.chunks(size, alignment):
                yield chunk
1342450
function(data) {
      var segment;
      log('json loaded');

      if (!data.files[0].length) {
        data.files = [data.files];
      }

      for (var i = 0, l = data.files.length; i < l; i++) {
        _files.push([]);
        segment = data.files[i];

        for (var m = 0, n = segment.length; m < n; m++) {
          arrangeData(i, m, segment[m]);
        }
      }
    }
912016
def merge(files, accumulate=True):
        
        metrics = {}
        for f in files:
            parts = os.path.basename(f).split('_')
            typ = parts[0]
            d = MmapedDict(f, read_mode=True)
            for key, value in d.read_all_values():
                metric_name, name, labels = json.loads(key)
                labels_key = tuple(sorted(labels.items()))

                metric = metrics.get(metric_name)
                if metric is None:
                    metric = Metric(metric_name, 'Multiprocess metric', typ)
                    metrics[metric_name] = metric

                if typ == 'gauge':
                    pid = parts[2][:-3]
                    metric._multiprocess_mode = parts[1]
                    metric.add_sample(name, labels_key + (('pid', pid),), value)
                else:
                    # The duplicates and labels are fixed in the next for.
                    metric.add_sample(name, labels_key, value)
            d.close()

        for metric in metrics.values():
            samples = defaultdict(float)
            buckets = {}
            for s in metric.samples:
                name, labels, value = s.name, s.labels, s.value
                if metric.type == 'gauge':
                    without_pid = tuple(l for l in labels if l[0] != 'pid')
                    if metric._multiprocess_mode == 'min':
                        current = samples.setdefault((name, without_pid), value)
                        if value < current:
                            samples[(s.name, without_pid)] = value
                    elif metric._multiprocess_mode == 'max':
                        current = samples.setdefault((name, without_pid), value)
                        if value > current:
                            samples[(s.name, without_pid)] = value
                    elif metric._multiprocess_mode == 'livesum':
                        samples[(name, without_pid)] += value
                    else:  # all/liveall
                        samples[(name, labels)] = value

                elif metric.type == 'histogram':
                    bucket = tuple(float(l[1]) for l in labels if l[0] == 'le')
                    if bucket:
                        # _bucket
                        without_le = tuple(l for l in labels if l[0] != 'le')
                        buckets.setdefault(without_le, {})
                        buckets[without_le].setdefault(bucket[0], 0.0)
                        buckets[without_le][bucket[0]] += value
                    else:
                        # _sum/_count
                        samples[(s.name, labels)] += value

                else:
                    # Counter and Summary.
                    samples[(s.name, labels)] += value

            # Accumulate bucket values.
            if metric.type == 'histogram':
                for labels, values in buckets.items():
                    acc = 0.0
                    for bucket, value in sorted(values.items()):
                        sample_key = (
                            metric.name + '_bucket',
                            labels + (('le', floatToGoString(bucket)),),
                        )
                        if accumulate:
                            acc += value
                            samples[sample_key] = acc
                        else:
                            samples[sample_key] = value
                    if accumulate:
                        samples[(metric.name + '_count', labels)] = acc

            # Convert to correct sample format.
            metric.samples = [Sample(name_, dict(labels), value) for (name_, labels), value in samples.items()]
        return metrics.values()
900688
def create_chunk(buf):
    """"""
    chunk = []
    chunk.append(s2b('{:X}\r\n'.format(len(buf))))
    chunk.append(buf)
    chunk.append(b'\r\n')
    return b''.join(chunk)
696501
def split_files(files, split_train, split_val, use_test):
    
    files_train = files[:split_train]
    files_val = files[split_train:split_val] if use_test else files[split_train:]

    li = [(files_train, 'train'), (files_val, 'val')]

    # optional test folder
    if use_test:
        files_test = files[split_val:]
        li.append((files_test, 'test'))
    return li
861782
def split_input(cls, mapper_spec):
    """"""
    filelists = mapper_spec.params[cls.FILES_PARAM]
    max_values_count = mapper_spec.params.get(cls.MAX_VALUES_COUNT_PARAM, -1)
    max_values_size = mapper_spec.params.get(cls.MAX_VALUES_SIZE_PARAM, -1)
    return [cls([0] * len(files), max_values_count, max_values_size)
            for files in filelists]
1332799
def _get_split_tasks(args, split_fn, file_key, outfile_i=-1):
    
    split_args = []
    combine_map = {}
    finished_map = collections.OrderedDict()
    extras = []
    for data in args:
        out_final, out_parts = split_fn(data)
        for parts in out_parts:
            split_args.append([utils.deepish_copy(data)] + list(parts))
        for part_file in [x[outfile_i] for x in out_parts]:
            combine_map[part_file] = out_final
        if len(out_parts) == 0:
            if out_final is not None:
                if out_final not in finished_map:
                    data[file_key] = out_final
                    finished_map[out_final] = [data]
                else:
                    extras.append([data])
            else:
                extras.append([data])
    return split_args, combine_map, list(finished_map.values()), extras