Create batches in expected AWS Cloudsearch format, limiting the
    byte size per batch according to given max_batch_size

    See: http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html
760923
def create_document_batches(jsonlines, id_field, max_batch_size=CLOUDSEARCH_MAX_BATCH_SIZE):
    
    batch = []
    fixed_initial_size = 2

    def create_entry(line):
        try:
            record = json.loads(line)
        except:
            raise ValueError('Could not parse JSON from: %s' % line)
        key = record[id_field]
        return '{"type":"add","id":%s,"fields":%s}' % (json.dumps(key), line)

    current_size = fixed_initial_size
    for line in jsonlines:
        entry = create_entry(line)
        entry_size = len(entry) + 1

        if max_batch_size > (current_size + entry_size):
            current_size += entry_size
            batch.append(entry)
        else:
            yield '[' + ','.join(batch) + ']'
            batch = [entry]
            current_size = fixed_initial_size + entry_size

    if batch:
        yield '[' + ','.join(batch) + ']'
221752
def minibatch_by_words(items, size, tuples=True, count_words=len):
    """"""
    if isinstance(size, int):
        size_ = itertools.repeat(size)
    else:
        size_ = size
    items = iter(items)
    while True:
        batch_size = next(size_)
        batch = []
        while batch_size >= 0:
            try:
                if tuples:
                    doc, gold = next(items)
                else:
                    doc = next(items)
            except StopIteration:
                if batch:
                    yield batch
                return
            batch_size -= count_words(doc)
            if tuples:
                batch.append((doc, gold))
            else:
                batch.append(doc)
        if batch:
            yield batch
554637
def batch_fn(dataset, training, shapes, target_names,
             batch_size=32, eval_batch_size=32, bucket_batch_length=32,
             bucket_max_length=256, bucket_min_length=8,
             bucket_length_step=1.1, buckets=None):
  """"""
  del target_names
  # If bucketing is not specified, check if target shapes are variable.
  cur_batch_size = batch_size if training else eval_batch_size
  if buckets is None:
    variable_target_shapes = False
    target_shape = shapes[1]
    for dim in target_shape:
      if dim is None:
        variable_target_shapes = True
    tf.logging.info("Heuristically setting bucketing to %s based on shapes "
                    "of target tensors." % variable_target_shapes)
    if variable_target_shapes:
      batch_size_per_token = cur_batch_size * bucket_batch_length
      scheme = data_reader.batching_scheme(batch_size_per_token,
                                           bucket_max_length,
                                           bucket_min_length,
                                           bucket_length_step,
                                           drop_long_sequences=training)
      buckets = (scheme["boundaries"], scheme["batch_sizes"])

  if buckets:
    tf.logging.info("Bucketing with buckets %s." % str(buckets))
    def example_length(_, target):
      return tf.shape(target)[0]
    boundaries, batch_sizes = buckets
    dataset = dataset.apply(tf.data.experimental.bucket_by_sequence_length(
        example_length, boundaries, batch_sizes))
  else:
    dataset = dataset.padded_batch(cur_batch_size, shapes)
  return dataset
1048079
def make_batch(size, graph):
        
        i = iter(graph)
        chunk = list(islice(i, size))
        while chunk:
            yield chunk
            chunk = list(islice(i, size))
399875
def encrypt_batch_items(table_name, aws_cmk_id):
    """"""
    index_keys = [
        {"partition_attribute": {"S": "is this"}, "sort_attribute": {"N": "55"}},
        {"partition_attribute": {"S": "is this"}, "sort_attribute": {"N": "56"}},
        {"partition_attribute": {"S": "is this"}, "sort_attribute": {"N": "57"}},
        {"partition_attribute": {"S": "another"}, "sort_attribute": {"N": "55"}},
    ]
    plaintext_additional_attributes = {
        "example": {"S": "data"},
        "some numbers": {"N": "99"},
        "and some binary": {"B": b"\x00\x01\x02"},
        "leave me": {"S": "alone"},  # We want to ignore this attribute
    }
    plaintext_items = []
    for key in index_keys:
        _attributes = key.copy()
        _attributes.update(plaintext_additional_attributes)
        plaintext_items.append(_attributes)

    # Collect all of the attributes that will be encrypted (used later).
    encrypted_attributes = set(plaintext_additional_attributes.keys())
    encrypted_attributes.remove("leave me")
    # Collect all of the attributes that will not be encrypted (used later).
    unencrypted_attributes = set(index_keys[0].keys())
    unencrypted_attributes.add("leave me")

    # Create a normal client.
    client = boto3.client("dynamodb")
    # Create a crypto materials provider using the specified AWS KMS key.
    aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id)
    # Create attribute actions that tells the encrypted client to encrypt all attributes except one.
    actions = AttributeActions(
        default_action=CryptoAction.ENCRYPT_AND_SIGN, attribute_actions={"leave me": CryptoAction.DO_NOTHING}
    )
    # Use these objects to create an encrypted client.
    encrypted_client = EncryptedClient(client=client, materials_provider=aws_kms_cmp, attribute_actions=actions)

    # Put the items to the table, using the encrypted client to transparently encrypt them.
    encrypted_client.batch_write_item(
        RequestItems={table_name: [{"PutRequest": {"Item": item}} for item in plaintext_items]}
    )

    # Get the encrypted item using the standard client.
    encrypted_items = client.batch_get_item(RequestItems={table_name: {"Keys": index_keys}})["Responses"][table_name]

    # Get the item using the encrypted client, transparently decyrpting it.
    decrypted_items = encrypted_client.batch_get_item(RequestItems={table_name: {"Keys": index_keys}})["Responses"][
        table_name
    ]

    def _select_index_from_item(item):
        """Find the index keys that match this item."""
        for index in index_keys:
            if all([item[key] == value for key, value in index.items()]):
                return index

        raise Exception("Index key not found in item.")

    def _select_item_from_index(index, all_items):
        """Find the item that matches these index keys."""
        for item in all_items:
            if all([item[key] == value for key, value in index.items()]):
                return item

        raise Exception("Index key not found in item.")

    for encrypted_item in encrypted_items:
        key = _select_index_from_item(encrypted_item)
        plaintext_item = _select_item_from_index(key, plaintext_items)
        decrypted_item = _select_item_from_index(key, decrypted_items)

        # Verify that all of the attributes are different in the encrypted item
        for name in encrypted_attributes:
            assert encrypted_item[name] != plaintext_item[name]
            assert decrypted_item[name] == plaintext_item[name]

        # Verify that all of the attributes that should not be encrypted were not.
        for name in unencrypted_attributes:
            assert decrypted_item[name] == encrypted_item[name] == plaintext_item[name]

    # Clean up the item
    encrypted_client.batch_write_item(
        RequestItems={table_name: [{"DeleteRequest": {"Key": key}} for key in index_keys]}
    )
552227
def batch(data, batch_size, batch_size_fn=None):
    """"""
    if batch_size_fn is None:
        def batch_size_fn(new, count, sofar):
            return count
    minibatch, size_so_far = [], 0
    for ex in data:
        minibatch.append(ex)
        size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)
        if size_so_far == batch_size:
            yield minibatch
            minibatch, size_so_far = [], 0
        elif size_so_far > batch_size:
            yield minibatch[:-1]
            minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)
    if minibatch:
        yield minibatch
967881
def batching(size):
    """"""

    if size < 1:
        raise ValueError("batching() size must be at least 1")

    def batching_transducer(reducer):
        return Batching(reducer, size)

    return batching_transducer
66648
def _batched_write_command_impl(
        namespace, operation, command, docs, check_keys, opts, ctx, buf):
    """"""
    max_bson_size = ctx.max_bson_size
    max_write_batch_size = ctx.max_write_batch_size
    # Max BSON object size + 16k - 2 bytes for ending NUL bytes.
    # Server guarantees there is enough room: SERVER-10643.
    max_cmd_size = max_bson_size + _COMMAND_OVERHEAD

    # No options
    buf.write(_ZERO_32)
    # Namespace as C string
    buf.write(b(namespace))
    buf.write(_ZERO_8)
    # Skip: 0, Limit: -1
    buf.write(_SKIPLIM)

    # Where to write command document length
    command_start = buf.tell()
    buf.write(bson.BSON.encode(command))

    # Start of payload
    buf.seek(-1, 2)
    # Work around some Jython weirdness.
    buf.truncate()
    try:
        buf.write(_OP_MAP[operation])
    except KeyError:
        raise InvalidOperation('Unknown command')

    if operation in (_UPDATE, _DELETE):
        check_keys = False

    # Where to write list document length
    list_start = buf.tell() - 4
    to_send = []
    idx = 0
    for doc in docs:
        # Encode the current operation
        key = b(str(idx))
        value = bson.BSON.encode(doc, check_keys, opts)
        # Is there enough room to add this document? max_cmd_size accounts for
        # the two trailing null bytes.
        enough_data = (buf.tell() + len(key) + len(value)) >= max_cmd_size
        enough_documents = (idx >= max_write_batch_size)
        if enough_data or enough_documents:
            if not idx:
                write_op = list(_FIELD_MAP.keys())[operation]
                _raise_document_too_large(
                    write_op, len(value), max_bson_size)
            break
        buf.write(_BSONOBJ)
        buf.write(key)
        buf.write(_ZERO_8)
        buf.write(value)
        to_send.append(doc)
        idx += 1

    # Finalize the current OP_QUERY message.
    # Close list and command documents
    buf.write(_ZERO_16)

    # Write document lengths and request id
    length = buf.tell()
    buf.seek(list_start)
    buf.write(_pack_int(length - list_start - 1))
    buf.seek(command_start)
    buf.write(_pack_int(length - command_start))

    return to_send, length
314997
def _batch_load(project, workspace, headerline, entity_data, chunk_size=500):
    


    if fcconfig.verbosity:
        print("Batching " + str(len(entity_data)) + " updates to Firecloud...")

    # Parse the entity type from the first cell, e.g. "entity:sample_id"
    # First check that the header is valid
    if not _valid_headerline(headerline):
        eprint("Invalid loadfile header:\n" + headerline)
        return 1

    update_type = "membership" if headerline.startswith("membership") else "entitie"
    etype = headerline.split('\t')[0].split(':')[1].replace("_id", "")

    # Split entity_data into chunks
    total = int(len(entity_data) / chunk_size) + 1
    batch = 0
    for i in range(0, len(entity_data), chunk_size):
        batch += 1
        if fcconfig.verbosity:
            print("Updating {0} {1}s {2}-{3}, batch {4}/{5}".format(
                etype, update_type, i+1, min(i+chunk_size, len(entity_data)),
                batch, total))
        this_data = headerline + '\n' + '\n'.join(entity_data[i:i+chunk_size])

        # Now push the entity data to firecloud
        r = fapi.upload_entities(project, workspace, this_data)
        fapi._check_response_code(r, 200)

    return 0
1162378
def batch!(operations, request_options = {})
      res = batch(operations, request_options)
      res['taskID'].each do |index, taskID|
        wait_task(index, taskID, WAIT_TASK_DEFAULT_TIME_BEFORE_RETRY, request_options)
      end
    end
274220
def setBatchSize(self, size):
        
        self._batchSize = size
        try:
            self._worker.setBatchSize(size)
        except AttributeError:
            pass
156944
def batch(self, requests):
        

        for request in requests:
            if 'body' in request:
                request['body'] = urlencode(request['body'])

        def _grouper(complete_list, n=1):
            """
            Batches a list into constant size chunks.

            :param complete_list: A input list (not a generator).
            :param n: The size of the chunk.

            Adapted from <http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python>
            """

            for i in range(0, len(complete_list), n):
                yield complete_list[i:i + n]

        responses = []

        # Maximum batch size for Facebook is 50 so split up requests
        # https://developers.facebook.com/docs/graph-api/making-multiple-requests/#limits
        for group in _grouper(requests, 50):
            responses += self.post(
                batch=json.dumps(group)
            )

        for response, request in zip(responses, requests):

            # Facilitate for empty Graph API responses.
            #
            # https://github.com/jgorset/facepy/pull/30
            if not response:
                yield None
                continue

            try:
                yield self._parse(response['body'])
            except FacepyError as exception:
                exception.request = request
                yield exception
577094
def batch_delete_s3_objects(
        keys,
        aws_bucket_name,
        chunk_size=100,
        s3_client=None
):
    
    if s3_client is None:
        s3_client, s3_resource = get_s3_client()

    key_chunks = []
    for i in range(0, len(keys), chunk_size):
        chunk = []
        for key in (list(keys)[i:i+100]):
            chunk.append({'Key': key})
        key_chunks.append(chunk)
    for chunk in key_chunks:
        s3_client.delete_objects(
            Bucket=aws_bucket_name,
            Delete={'Objects': chunk}
        )
1071362
def _bulk_history_create(self, model, batch_size):
        

        instances = []
        history = utils.get_history_manager_for_model(model)
        if self.verbosity >= 1:
            self.stdout.write(
                "Starting bulk creating history models for {} instances {}-{}".format(
                    model, 0, batch_size
                )
            )

        iterator_kwargs = (
            {"chunk_size": batch_size} if django.VERSION >= (2, 0, 0) else {}
        )
        for index, instance in enumerate(
            model._default_manager.iterator(**iterator_kwargs)
        ):
            # Can't Just pass batch_size to bulk_create as this can lead to
            # Out of Memory Errors as we load too many models into memory after
            # creating them. So we only keep batch_size worth of models in
            # historical_instances and clear them after we hit batch_size
            if index % batch_size == 0:

                history.bulk_history_create(instances, batch_size=batch_size)

                instances = []

                if self.verbosity >= 1:
                    self.stdout.write(
                        "Finished bulk creating history models for {} "
                        "instances {}-{}, starting next {}".format(
                            model, index - batch_size, index, batch_size
                        )
                    )

            instances.append(instance)

        # create any we didn't get in the last loop
        if instances:
            history.bulk_history_create(instances, batch_size=batch_size)
1030189
public static BatchedWriteBehindConfigurationBuilder newBatchedWriteBehindConfiguration(long maxDelay, TimeUnit maxDelayUnit, int batchSize) {
    return new BatchedWriteBehindConfigurationBuilder(maxDelay, maxDelayUnit, batchSize);
  }
1243626
def chunks(iterable, size=50):
    """"""
    batch = []
    for n in iterable:
        batch.append(n)
        if len(batch) % size == 0:
            yield batch
            batch = []
    if batch:
        yield batch
628803
private void getBatchSize(String persistenceUnit, Map<String, Object> puProperties)
    {
        String batch_Size = puProperties != null ? (String) puProperties.get(PersistenceProperties.KUNDERA_BATCH_SIZE)
                : null;
        if (batch_Size != null)
        {
            setBatchSize(Integer.valueOf(batch_Size));
        }
        else
        {
            PersistenceUnitMetadata puMetadata = KunderaMetadataManager.getPersistenceUnitMetadata(kunderaMetadata,
                    persistenceUnit);
            setBatchSize(puMetadata.getBatchSize());
        }
    }