Compute split-per-bucket (single split for all shards in a bucket).
509773
private BucketShards computeMerged()
            throws SQLException
    {
        if (resultSet.isAfterLast()) {
            return endOfData();
        }
        if (first) {
            first = false;
            if (!resultSet.next()) {
                return endOfData();
            }
        }

        int bucketNumber = resultSet.getInt("bucket_number");
        ImmutableSet.Builder<ShardNodes> shards = ImmutableSet.builder();

        do {
            UUID shardUuid = uuidFromBytes(resultSet.getBytes("shard_uuid"));
            int bucket = resultSet.getInt("bucket_number");
            Set<String> nodeIdentifiers = ImmutableSet.of(getBucketNode(bucket));

            shards.add(new ShardNodes(shardUuid, nodeIdentifiers));
        }
        while (resultSet.next() && resultSet.getInt("bucket_number") == bucketNumber);

        return new BucketShards(OptionalInt.of(bucketNumber), shards.build());
    }
1059678
def split(self, amount):
        
        split_objs = list(self.all())
        if not split_objs:
            raise NoSplitsFoundForRecurringCost()

        portions = [split_obj.portion for split_obj in split_objs]

        split_amounts = ratio_split(amount, portions)
        return [
            (split_objs[i], split_amount)
            for i, split_amount
            in enumerate(split_amounts)
        ]
745254
def _split_current_epoch(self):
    
    num_frames = self._calc_num_frames(self._current_epoch_rollouts)
    num_shards = sum(split["shards"] for split in self.dataset_splits)
    shard_size = num_frames // num_shards

    splits = self.dataset_splits
    num_saved_frames = 0
    split_index = 0
    split_begin_index = 0
    rollouts_by_split = collections.defaultdict(list)

    def split_size(split_index):
      return splits[split_index]["shards"] * shard_size

    for rollout in self._current_epoch_rollouts:
      num_saved_frames_current_rollout = 0
      # Split the rollout into chunks corresponding to dataset splits. In most
      # cases there should be only one chunk. On dataset split boundary there
      # will be two. If a rollout is longer then the size of a dataset split,
      # there might be more.
      while num_saved_frames_current_rollout < len(rollout):
        max_chunk_length = (
            split_begin_index + split_size(split_index) - num_saved_frames
        )
        if split_index == len(splits) - 1:
          # Put the remainder in the last split to preserve the ordering.
          max_chunk_length = len(rollout)
        rollout_chunk = rollout[
            num_saved_frames_current_rollout:
            (num_saved_frames_current_rollout + max_chunk_length)
        ]
        rollouts_by_split[splits[split_index]["split"]].append(rollout_chunk)
        num_saved_frames_current_rollout += len(rollout_chunk)
        num_saved_frames += len(rollout_chunk)

        if num_saved_frames == split_begin_index + split_size(split_index):
          split_begin_index += split_size(split_index)
          split_index = min(split_index + 1, len(splits) - 1)

    self._rollouts_by_epoch_and_split[self.current_epoch] = rollouts_by_split
    self._current_epoch_rollouts = []
520143
def dataset_splits(self):
    """"""
    return [{
        "split": problem.DatasetSplit.TRAIN,
        "shards": self.num_train_shards,
    }, {
        "split": problem.DatasetSplit.EVAL,
        "shards": self.num_eval_shards,
    }, {
        "split": problem.DatasetSplit.TEST,
        "shards": self.num_test_shards,
    }]
1057838
def sort_and_distribute(array, splits=2):
   
   if not isinstance(array, (list,tuple)): raise TypeError("array must be a list")
   if not isinstance(splits, int): raise TypeError("splits must be an integer")
   remaining = sorted(array)
   if sys.version_info < (3, 0):
      myrange = xrange(splits)
   else:
      myrange = range(splits)
   groups = [[] for i in myrange]
   while len(remaining) > 0:
      for i in myrange:
         if len(remaining) > 0: groups[i].append(remaining.pop(0))
   return groups
872275
@Override
    public java.util.concurrent.Future<SplitShardResult> splitShardAsync(String streamName, String shardToSplit, String newStartingHashKey) {

        return splitShardAsync(new SplitShardRequest().withStreamName(streamName).withShardToSplit(shardToSplit).withNewStartingHashKey(newStartingHashKey));
    }
1175768
def _split_one_shard_to_multiple(client, project, logstore, shard_info, count, current_shard_count):
    
    distance = shard_info['length'] // count
    if distance <= 0 or count <= 1:
        return [shard_info['info']], 0

    rw_shards, increased_shard_count = {shard_info['id']: shard_info['info']}, 0
    for x in range(1, count):
        new_hash = shard_info['start'] + distance * x
        new_hash = hex(new_hash)[2:].strip('lL')
        new_hash = '0' * (TOTAL_HASH_LENGTH - len(new_hash)) + new_hash
        try:
            if x == 1:
                res = client.split_shard(project, logstore, shard_info['id'], new_hash)
            else:
                res = client.split_shard(project, logstore, current_shard_count - 1, new_hash)

            # new rw_shards
            for shard in res.shards:
                if shard['status'] == 'readonly':
                    del rw_shards[shard['shardID']]
                else:
                    rw_shards[shard['shardID']] = shard

            current_shard_count += res.count - 1
            increased_shard_count += res.count - 1
            logger.info("split shard: project={0}, logstore={1}, shard_info={2}, count={3}, current_shard_count={4}".format(project, logstore, shard_info, count, current_shard_count))
        except Exception as ex:
            print(ex)
            print(x, project, logstore, shard_info, count, current_shard_count)
            raise

    return rw_shards.values(), increased_shard_count
205746
def split(items, n)
      if n == 1
        [items]
      elsif items.length <= n
        Array.new(n) {|i| items[i] ? [items[i]] : [] }
      else
        min_max = nil
        min_arr = nil
        i = 0
        while i <= items.length-n
          i += 1
          # Try placing 1, 2, 3, ... items to first chunk.
          # Calculate the remaining chunks recursively.
          cols = [items[0,i]] + split(items[i, items.length], n-1)
          max = max_sum(cols)
          # Is this the optimal solution so far? Remember it.
          if !min_max || max < min_max
            min_max = max
            min_arr = cols
          end
        end
        min_arr
      end
    end
767677
def partition(a, sz): 
    """"""
    return [a[i:i+sz] for i in range(0, len(a), sz)]
1141985
@Override
    public java.util.concurrent.Future<SplitShardResult> splitShardAsync(String streamName, String shardToSplit, String newStartingHashKey,
            com.amazonaws.handlers.AsyncHandler<SplitShardRequest, SplitShardResult> asyncHandler) {

        return splitShardAsync(new SplitShardRequest().withStreamName(streamName).withShardToSplit(shardToSplit).withNewStartingHashKey(newStartingHashKey),
                asyncHandler);
    }
435581
def dataset_splits(self):
    """"""
    return [{
        "split": problem.DatasetSplit.TRAIN,
        "shards": _TRAIN_SHARDS,
    }, {
        "split": problem.DatasetSplit.EVAL,
        "shards": _DEV_SHARDS,
    }]
861782
def split_input(cls, mapper_spec):
    """"""
    filelists = mapper_spec.params[cls.FILES_PARAM]
    max_values_count = mapper_spec.params.get(cls.MAX_VALUES_COUNT_PARAM, -1)
    max_values_size = mapper_spec.params.get(cls.MAX_VALUES_SIZE_PARAM, -1)
    return [cls([0] * len(files), max_values_count, max_values_size)
            for files in filelists]
95568
def check_splits_equals(splits1, splits2):
  """"""
  if set(splits1) ^ set(splits2):  # Name intersection should be null
    return False
  for _, (split1, split2) in utils.zip_dict(splits1, splits2):
    if split1.num_shards != split2.num_shards:
      return False
  return True
413672
def compute_n_splits(cv, X, y=None, groups=None):
    
    if not any(is_dask_collection(i) for i in (X, y, groups)):
        return cv.get_n_splits(X, y, groups)

    if isinstance(cv, (_BaseKFold, BaseShuffleSplit)):
        return cv.n_splits

    elif isinstance(cv, PredefinedSplit):
        return len(cv.unique_folds)

    elif isinstance(cv, _CVIterableWrapper):
        return len(cv.cv)

    elif isinstance(cv, (LeaveOneOut, LeavePOut)) and not is_dask_collection(X):
        # Only `X` is referenced for these classes
        return cv.get_n_splits(X, None, None)

    elif isinstance(cv, (LeaveOneGroupOut, LeavePGroupsOut)) and not is_dask_collection(
        groups
    ):
        # Only `groups` is referenced for these classes
        return cv.get_n_splits(None, None, groups)

    else:
        return delayed(cv).get_n_splits(X, y, groups).compute()
749224
def split_input(cls, mapper_spec):
    
    params = _get_params(mapper_spec)
    shard_count = mapper_spec.shard_count

    # Pick out the overall start and end times and time step per shard.
    start_time = params[cls.START_TIME_PARAM]
    end_time = params[cls.END_TIME_PARAM]
    seconds_per_shard = (end_time - start_time) / shard_count

    # Create a LogInputReader for each shard, modulating the params as we go.
    shards = []
    for _ in xrange(shard_count - 1):
      params[cls.END_TIME_PARAM] = (params[cls.START_TIME_PARAM] +
                                    seconds_per_shard)
      shards.append(LogInputReader(**params))
      params[cls.START_TIME_PARAM] = params[cls.END_TIME_PARAM]

    # Create a final shard to complete the time range.
    params[cls.END_TIME_PARAM] = end_time
    return shards + [LogInputReader(**params)]
1151372
def _partition(s, sep, find):
    
    idx = find(sep)
    if idx != -1:
        left = s[0:idx]
        return left, sep, s[len(left)+len(sep):]
387921
def partition_pair(bif_point):
    ''''''
    n = float(sum(1 for _ in bif_point.children[0].ipreorder()))
    m = float(sum(1 for _ in bif_point.children[1].ipreorder()))
    return (n, m)