Batch up pre-planned fragments, but handle ad hoc independently.
169051
private VoltTable[] fastPath(List<QueuedSQL> batch, final boolean finalTask) {
        final int batchSize = batch.size();
        Object[] params = new Object[batchSize];
        long[] fragmentIds = new long[batchSize];
        String[] sqlTexts = new String[batchSize];
        boolean[] isWriteFrag = new boolean[batchSize];
        int[] sqlCRCs = new int[batchSize];
        int succeededFragmentsCount = 0;

        int i = 0;
        for (final QueuedSQL qs : batch) {
            assert (qs.stmt.collector == null);
            fragmentIds[i] = qs.stmt.aggregator.id;
            // use the pre-serialized params if it exists
            params[i] = qs.params;
            sqlTexts[i] = qs.stmt.getText();
            isWriteFrag[i] = !qs.stmt.isReadOnly;
            sqlCRCs[i] = SQLStmtAdHocHelper.getHash(qs.stmt);
            i++;
        }

        VoltTable[] results = null;
        // Before executing the fragments, tell the EE if this batch should be
        // timed.
        getExecutionEngine().setPerFragmentTimingEnabled(m_perCallStats.samplingStmts());
        try {
            FastDeserializer fragResult = m_site.executePlanFragments(batchSize, fragmentIds, null, params,
                    m_determinismHash, sqlTexts, isWriteFrag, sqlCRCs, m_txnState.txnId, m_txnState.m_spHandle,
                    m_txnState.uniqueId, m_isReadOnly, VoltTrace.log(VoltTrace.Category.EE) != null);
            final int totalSize;
            try {
                // read the size of the DR buffer used
                fragResult.readInt();
                // read the complete size of the buffer used
                totalSize = fragResult.readInt();
            } catch (final IOException ex) {
                log.error("Failed to deserialze result table" + ex);
                throw new EEException(ExecutionEngine.ERRORCODE_WRONG_SERIALIZED_BYTES);
            }
            final ByteBuffer rawDataBuff;
            if ((m_batchIndex == 0 && !m_site.usingFallbackBuffer()) || finalTask) {
                // If this is the first or final batch, skip the copy of the
                // underlying byte array
                rawDataBuff = fragResult.buffer();
            } else {
                rawDataBuff = fragResult.readBuffer(totalSize);
            }
            results = TableHelper.convertBackedBufferToTables(rawDataBuff, batchSize);
        } catch (Throwable ex) {
            if (!m_isReadOnly) {
                // roll back the current batch and re-throw the EE exception
                m_site.truncateUndoLog(true, false,
                        m_spBigBatchBeginToken >= 0 ? m_spBigBatchBeginToken : m_site.getLatestUndoToken(),
                        m_txnState.m_spHandle, null);
            }

            throw ex;
        } finally {
            long[] executionTimes = null;
            if (m_perCallStats.samplingStmts()) {
                executionTimes = new long[batchSize];
            }
            succeededFragmentsCount = getExecutionEngine().extractPerFragmentStats(batchSize, executionTimes);

            for (i = 0; i < batchSize; i++) {
                QueuedSQL qs = batch.get(i);
                // No coordinator task for a single partition procedure.
                boolean isCoordinatorTask = false;
                // If all the fragments in this batch are executed successfully,
                // succeededFragmentsCount == batchSize.
                // Otherwise, the fragment whose index equals
                // succeededFragmentsCount is the one that failed.
                boolean failed = i == succeededFragmentsCount;

                m_perCallStats.recordStatementStats(qs.stmt.getStmtName(), isCoordinatorTask, failed,
                        executionTimes == null ? 0 : executionTimes[i], results == null ? null : results[i], qs.params);

                // If this fragment failed, no subsequent fragments will be
                // executed.
                if (failed) {
                    break;
                }
            }
        }
        return results;
    }
964718
def _teardown(self):
        ""
        self.example.after(self.context)
        if self.is_root_runner:
            run.after_all.execute(self.context)
        #self.context = self.context._parent
        self.has_ran = True
90814
def batchify(data, batch_size):
    """"""
    nbatch = data.shape[0] // batch_size
    data = data[:nbatch * batch_size]
    data = data.reshape((batch_size, nbatch)).T
    return data
1162378
def batch!(operations, request_options = {})
      res = batch(operations, request_options)
      res['taskID'].each do |index, taskID|
        wait_task(index, taskID, WAIT_TASK_DEFAULT_TIME_BEFORE_RETRY, request_options)
      end
    end
1211265
def pre_run(self, outline=False, *args, **kwargs):
        """"""
        pre_destroy = self.context.config.pre_destroy
        if not outline and pre_destroy:
            handle_hooks(
                stage="pre_destroy",
                hooks=pre_destroy,
                provider=self.provider,
                context=self.context)
865365
def process_all(self):
        """"""
        results = []
        veto_info = []
        while 1:
            result, veto = self._process_batch()
            if result is False: return False
            if result is None: break
            results.append(result)
            veto_info += veto

        result = self.combine_results(results)

        if self.max_triggers_in_batch:
            sort = result['snr'].argsort()[::-1][:self.max_triggers_in_batch]
            for key in result:
                result[key] = result[key][sort]

            tmp = veto_info
            veto_info = [tmp[i] for i in sort]

        result = self._process_vetoes(result, veto_info)
        return result
744506
public void addCustomFragment(byte[] planHash, int outputDepId, ByteBuffer parameterSet, byte[] fragmentPlan, String stmtText) {
        FragmentData item = new FragmentData();
        item.m_planHash = planHash;
        item.m_outputDepId = outputDepId;
        item.m_parameterSet = parameterSet;
        item.m_fragmentPlan = fragmentPlan;
        item.m_stmtText = stmtText.getBytes();
        m_items.add(item);
    }
1170062
private void fireOnRecoverBasic(ChannelEvent e) {
        List<EventListener> listeners = changes.getListenerList(AMQP);
        for (EventListener listener : listeners) {
            ChannelListener amqpListener = (ChannelListener)listener;
            amqpListener.onRecoverBasic(e);
        }
    }
1352186
public void consolidate() {
		Decl.Module module = target.getModule();
		for(ExternalGroup group : consolidations) {
			// TODO: this could be made way more efficient by collecting all consolidate
			// units into one batch
			module.putExtern(group.consolidate());
		}
		consolidations.clear();
	}
1170996
function proceedWithBackup() {
    if (options.resume) {
      // pick up from existing log file from previous run
      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
    } else {
      // create new log file and process
      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
        if (err) {
          ee.emit('error', err);
        } else {
          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
        }
      });
    }
  }
66527
def _group_batches_shared(xs, caller_batch_fn, prep_data_fn):
    
    singles = []
    batch_groups = collections.defaultdict(list)
    for args in xs:
        data = utils.to_single_data(args)
        caller, batch = caller_batch_fn(data)
        region = _list_to_tuple(data["region"]) if "region" in data else ()
        if batch is not None:
            batches = batch if isinstance(batch, (list, tuple)) else [batch]
            for b in batches:
                batch_groups[(b, region, caller)].append(utils.deepish_copy(data))
        else:
            data = prep_data_fn(data, [data])
            singles.append(data)
    batches = []
    for batch, items in batch_groups.items():
        batch_data = utils.deepish_copy(_pick_lead_item(items))
        # For nested primary batches, split permanently by batch
        if tz.get_in(["metadata", "batch"], batch_data):
            batch_name = batch[0]
            batch_data["metadata"]["batch"] = batch_name
        batch_data = prep_data_fn(batch_data, items)
        batch_data["group_orig"] = _collapse_subitems(batch_data, items)
        batch_data["group"] = batch
        batches.append(batch_data)
    return singles + batches
79128
void handleFragmentTaskMessage(FragmentTaskMessage message)
    {
        FragmentTaskMessage msg = message;
        long newSpHandle;
        //The site has been marked as non-leader. The follow-up batches or fragments are processed here
        if (!message.isForReplica() && (m_isLeader || message.isExecutedOnPreviousLeader())) {
            // message processed on leader
            // Quick hack to make progress...we need to copy the FragmentTaskMessage
            // before we start mucking with its state (SPHANDLE).  We need to revisit
            // all the messaging mess at some point.
            msg = new FragmentTaskMessage(message.getInitiatorHSId(),
                    message.getCoordinatorHSId(), message);
            //Not going to use the timestamp from the new Ego because the multi-part timestamp is what should be used
            msg.setTimestamp(message.getTimestamp());
            msg.setExecutedOnPreviousLeader(message.isExecutedOnPreviousLeader());
            if (!message.isReadOnly()) {
                TxnEgo ego = advanceTxnEgo();
                newSpHandle = ego.getTxnId();

                if (m_outstandingTxns.get(msg.getTxnId()) == null) {
                    updateMaxScheduledTransactionSpHandle(newSpHandle);
                }
            } else {
                newSpHandle = getMaxScheduledTxnSpHandle();
            }

            msg.setSpHandle(newSpHandle);
            msg.setLastSpUniqueId(m_uniqueIdGenerator.getLastUniqueId());
            logRepair(msg);
            if (msg.getInitiateTask() != null) {
                msg.getInitiateTask().setSpHandle(newSpHandle);//set the handle
                //Trigger reserialization so the new handle is used
                msg.setStateForDurability(msg.getInitiateTask(), msg.getInvolvedPartitions());
            }

            /*
             * If there a replicas to send it to, forward it!
             * Unless... it's read only AND not a sysproc. Read only sysprocs may expect to be sent
             * everywhere.
             * In that case don't propagate it to avoid a determinism check and extra messaging overhead
             */
            if (IS_KSAFE_CLUSTER && (!message.isReadOnly() || msg.isSysProcTask())) {
                for (long hsId : m_sendToHSIds) {
                    FragmentTaskMessage finalMsg = msg;
                    final VoltTrace.TraceEventBatch traceLog = VoltTrace.log(VoltTrace.Category.SPI);
                    if (traceLog != null) {
                        traceLog.add(() -> VoltTrace.beginAsync("replicatefragment",
                                                                MiscUtils.hsIdPairTxnIdToString(m_mailbox.getHSId(), hsId, finalMsg.getSpHandle(), finalMsg.getTxnId()),
                                                                "txnId", TxnEgo.txnIdToString(finalMsg.getTxnId()),
                                                                "dest", CoreUtils.hsIdToString(hsId)));
                    }
                }

                FragmentTaskMessage replmsg =
                    new FragmentTaskMessage(m_mailbox.getHSId(),
                            m_mailbox.getHSId(), msg);
                replmsg.setForReplica(true);
                replmsg.setTimestamp(msg.getTimestamp());
                // K-safety cluster doesn't always mean partition has replicas,
                // node failure may reduce the number of replicas for each partition.
                if (m_sendToHSIds.length > 0) {
                    m_mailbox.send(m_sendToHSIds,replmsg);
                }
                DuplicateCounter counter;
                /*
                 * Non-determinism should be impossible to happen with MP fragments.
                 * if you see "MP_DETERMINISM_ERROR" as procedure name in the crash logs
                 * something has horribly gone wrong.
                 */
                if (message.getFragmentTaskType() != FragmentTaskMessage.SYS_PROC_PER_SITE) {
                    counter = new DuplicateCounter(
                            msg.getCoordinatorHSId(),
                            msg.getTxnId(),
                            m_replicaHSIds,
                            replmsg);
                }
                else {
                    counter = new SysProcDuplicateCounter(
                            msg.getCoordinatorHSId(),
                            msg.getTxnId(),
                            m_replicaHSIds,
                            replmsg);
                }
                safeAddToDuplicateCounterMap(new DuplicateCounterKey(message.getTxnId(), newSpHandle), counter);
            }
        } else {
            // message processed on replica
            logRepair(msg);
            newSpHandle = msg.getSpHandle();
            setMaxSeenTxnId(newSpHandle);
        }

        Iv2Trace.logFragmentTaskMessage(message, m_mailbox.getHSId(), newSpHandle, false);
        doLocalFragmentOffer(msg);
    }
751603
def _split_batches(self, data, batch_size):
        """"""
        for i in range(0, len(data), batch_size):
            yield data[i : i + batch_size]
82379
private void handleFragmentGrouping(IReaction rxn, CxSmilesState cxstate) {
        // repartition/merge fragments
        if (cxstate.fragGroups != null) {

            final int reactant = 1;
            final int agent    = 2;
            final int product  = 3;

            // note we don't use a list for fragmap as the indexes need to stay consistent
            Map<Integer,IAtomContainer> fragMap = new LinkedHashMap<>();
            Map<IAtomContainer,Integer> roleMap = new HashMap<>();

            for (IAtomContainer mol : rxn.getReactants().atomContainers()) {
                fragMap.put(fragMap.size(), mol);
                roleMap.put(mol, reactant);
            }
            for (IAtomContainer mol : rxn.getAgents().atomContainers()) {
                fragMap.put(fragMap.size(), mol);
                roleMap.put(mol, agent);
            }
            for (IAtomContainer mol : rxn.getProducts().atomContainers()) {
                fragMap.put(fragMap.size(), mol);
                roleMap.put(mol, product);
            }

            // check validity of group
            boolean invalid = false;
            Set<Integer> visit = new HashSet<>();

            for (List<Integer> grouping : cxstate.fragGroups) {
                IAtomContainer dest = fragMap.get(grouping.get(0));
                if (dest == null)
                    continue;
                if (!visit.add(grouping.get(0)))
                    invalid = true;
                for (int i = 1; i < grouping.size(); i++) {
                    if (!visit.add(grouping.get(i)))
                        invalid = true;
                    IAtomContainer src = fragMap.get(grouping.get(i));
                    if (src != null) {
                        dest.add(src);
                        roleMap.put(src, 0); // no-role
                    }
                }
            }

            if (!invalid) {
                rxn.getReactants().removeAllAtomContainers();
                rxn.getAgents().removeAllAtomContainers();
                rxn.getProducts().removeAllAtomContainers();
                for (IAtomContainer mol : fragMap.values()) {
                    switch (roleMap.get(mol)) {
                        case reactant:
                            rxn.getReactants().addAtomContainer(mol);
                            break;
                        case product:
                            rxn.getProducts().addAtomContainer(mol);
                            break;
                        case agent:
                            rxn.getAgents().addAtomContainer(mol);
                            break;
                    }
                }
            }
        }
    }
319250
def _create_batches(self, instances: Iterable[Instance], shuffle: bool) -> Iterable[Batch]:
        
        raise NotImplementedError
1007089
def before_processing(eng, objects):
        """"""
        super(InvenioProcessingFactory, InvenioProcessingFactory)\
            .before_processing(eng, objects)
        eng.save(WorkflowStatus.RUNNING)
        db.session.commit()
247287
def batch(self):
        
        if self._no_kc:
            raise ValueError("Already in a batch")
        self._no_kc = True
        yield
        self._no_kc = False