Initializes batches by reading from the datastore.
166513
def init_from_datastore(self):
    """"""
    self._data = {}
    for entity in self._datastore_client.query_fetch(
        kind=self._entity_kind_batches):
      batch_id = entity.key.flat_path[-1]
      self._data[batch_id] = dict(entity)
      self._data[batch_id]['images'] = {}
    for entity in self._datastore_client.query_fetch(
        kind=self._entity_kind_images):
      batch_id = entity.key.flat_path[-3]
      image_id = entity.key.flat_path[-1]
      self._data[batch_id]['images'][image_id] = dict(entity)
237927
protected void batchHandle(Iterable<?> entities, BatchType batchType,
			int batchSize) {
		Session session = getCurrentSession();
		Iterator<?> it = entities.iterator();
		int count = 0;
		while (it.hasNext()) {
			count++;
			Object entity = it.next();
			switch (batchType) {
			case SAVE:
				session.save(entity);
				break;
			case UPDATE:
				session.update(entity);
				break;
			case SAVE_OR_UPDATE:
				session.saveOrUpdate(entity);
				break;
			case DELETE:
				session.delete(entity);
				break;
			}
			if (count % batchSize == 0) {
				session.flush();
				session.clear();
			}
		}
	}
1122260
protected function initialize()
    {
        if (!is_array($this->queryResult)) {
            $this->queryResult = $this->dataMapper->mapToObjects($this->persistenceManager->getObjectDataByQuery($this->query));
        }
    }
83846
def read_batch_from_datastore(self, class_batch_id):
    """"""
    client = self._datastore_client
    key = client.key(KIND_CLASSIFICATION_BATCH, class_batch_id)
    result = client.get(key)
    if result is not None:
      return dict(result)
    else:
      raise KeyError(
          'Key {0} not found in the datastore'.format(key.flat_path))
280177
def _init_datastores():
    
    global _DATASTORES
    array = settings.DATASTORES
    for config in array:
        cls = _lookup(config['ENGINE'])
        ds = _get_datastore(cls, DataStore, config)
        _DATASTORES.append(ds)
    legacy_settings = getattr(settings, 'MACHINE_CATEGORY_DATASTORES', None)
    if legacy_settings is not None:
        warnings.warn(
            "MACHINE_CATEGORY_DATASTORES is deprecated, "
            "please change to use DATASTORES",
        )
        for name in ['ldap']:
            array = settings.MACHINE_CATEGORY_DATASTORES.get(name, [])
            for config in array:
                cls = _lookup(config['ENGINE'])
                ds = _get_datastore(cls, DataStore, config)
                _DATASTORES.append(ds)
902072
def _initialize(self):
        
        if not os.path.exists(self._logdir):
            raise CollectorError("Log directory %s not exists" % self._logdir)

        self.logger.info("Collector started, taking %s as parent directory"
                         "for all job logs." % self._logdir)

        # clear old records
        JobRecord.objects.filter().delete()
        TrialRecord.objects.filter().delete()
        ResultRecord.objects.filter().delete()
368683
private void init() {
        if (databasePath.toFile().exists()) {
            // we have to initialize chunk size and the map of chunks
            // and return
            initChunkMap();
            int chunkSize = readChunkSize();
            assert(this.chunkSize == chunkSize);
            if (chunkSize == -1) {
                throw new InternalError("Impossible to read chunk size from the database '" + databasePath + "'");
            }
            this.chunkSize = chunkSize;
            logger.debug("Database was initialized previously. Nothing to do.");
            return;
        }

        Statement stmt = null;
        Connection connection = null;

        try {
            Class.forName("org.sqlite.JDBC");
            connection = DriverManager.getConnection("jdbc:sqlite:" + databasePath);

            // Create tables
            stmt = connection.createStatement();
            String sql = "CREATE TABLE chunk "
                    + "(id INTEGER PRIMARY KEY AUTOINCREMENT,"
                    + "chunk_id VARCHAR NOT NULL,"
                    + "chromosome VARCHAR NOT NULL, "
                    + "start INTEGER NOT NULL, "
                    + "end INTEGER NOT NULL); "
                    + "CREATE UNIQUE INDEX chunk_id_idx ON chunk (chunk_id);"
                    + "CREATE INDEX chrom_start_end_idx ON chunk (chromosome, start, end);";
            stmt.executeUpdate(sql);

            sql = "CREATE TABLE info "
                    + "(id INTEGER PRIMARY KEY AUTOINCREMENT,"
                    + "name VARCHAR NOT NULL,"
                    + "value VARCHAR NOT NULL);"
                    + "CREATE UNIQUE INDEX name_idx ON info (name);";
            stmt.executeUpdate(sql);

            sql = "insert into info (name, value) values ('chunkSize', '" + chunkSize + "');";
            stmt.executeUpdate(sql);

            sql = "CREATE TABLE file "
                    + "(id INTEGER PRIMARY KEY AUTOINCREMENT,"
                    + "path VARCHAR NOT NULL,"
                    + "name VARCHAR NOT NULL);"
                    + "CREATE UNIQUE INDEX path_idx ON file (path);";
            stmt.executeUpdate(sql);

            sql = "CREATE TABLE mean_coverage "
                    + "(chunk_id INTEGER,"
                    + "file_id INTEGER,"
                    + "v1 INTEGER, "
                    + "v2 INTEGER, "
                    + "v3 INTEGER, "
                    + "v4 INTEGER, "
                    + "v5 INTEGER, "
                    + "v6 INTEGER, "
                    + "v7 INTEGER, "
                    + "v8 INTEGER,"
                    + "PRIMARY KEY(chunk_id, file_id));";
            stmt.executeUpdate(sql);

            // close both resources
            stmt.close();
            connection.close();
        } catch (SQLException e) {
            logger.error(e.getClass().getName() + ": " + e.getMessage());
            cleanConnectionClose(connection, stmt);
        } catch (ClassNotFoundException e) {
            logger.error(e.getClass().getName() + ": " + e.getMessage());
            e.printStackTrace();
        }

        logger.debug("Initialized database successfully");
    }
133754
def _init_offsets(self, batchsize):
        
        upper_offsets = previous_lower_offsets = self._lower_offsets
        if not upper_offsets:
            upper_offsets = self.latest_offsets
        self._upper_offsets = {p: o for p, o in upper_offsets.items() if o > self._min_lower_offsets[p]}

        # remove db dupes not used anymore
        if self._dupes:
            for p in list(six.iterkeys(self._dupes)):
                if p not in self._upper_offsets:
                    db = self._dupes.pop(p)
                    db.close()
                    os.remove(db.filename)

        partition_batchsize = 0
        if self._upper_offsets:
            partition_batchsize = max(int(batchsize * self.__scan_excess), batchsize)
            self._lower_offsets = self._upper_offsets.copy()
            total_offsets_run = 0
            for p in sorted(self._upper_offsets.keys()):
                # readjust partition_batchsize when a partition scan starts from latest offset
                if total_offsets_run > 0 and partition_batchsize > batchsize:
                    partition_batchsize = batchsize
                if partition_batchsize > 0:
                    self._lower_offsets[p] = max(self._upper_offsets[p] - partition_batchsize, self._min_lower_offsets[p])
                    offsets_run = self._upper_offsets[p] - self._lower_offsets[p]
                    total_offsets_run += offsets_run
                    partition_batchsize = partition_batchsize - offsets_run
                else:
                    break
            log.info('Offset run: %d', total_offsets_run)
            # create new consumer if partition list changes
            if previous_lower_offsets is not None and set(previous_lower_offsets.keys()) != set(self._lower_offsets):
                self._create_scan_consumer(self._lower_offsets.keys())

            # consumer must restart from newly computed lower offsets
            self._update_offsets(self._lower_offsets)
        log.info('Initial offsets for topic %s: %s', self._topic, repr(self._lower_offsets))
        log.info('Target offsets for topic %s: %s', self._topic, repr(self._upper_offsets))

        return batchsize
748206
private void initializeChunkArtifacts() {
        String sourceMethod = "initializeChunkArtifacts";
        if (logger.isLoggable(Level.FINE))
            logger.entering(sourceClass, sourceMethod);

        ItemReader itemReader = chunk.getReader();
        List<Property> itemReaderProps = itemReader.getProperties() == null ? null : itemReader.getProperties().getPropertyList();
        try {
            InjectionReferences injectionRef = new InjectionReferences(runtimeWorkUnitExecution.getWorkUnitJobContext(), runtimeStepExecution, itemReaderProps);

            readerProxy = ProxyFactory.createItemReaderProxy(itemReader.getRef(), injectionRef, runtimeStepExecution);

            if (logger.isLoggable(Level.FINE)) {
                logger.fine("Created ItemReaderProxy for " + itemReader.getRef());
            }
        } catch (ArtifactValidationException e) {
            throw new BatchContainerServiceException("Cannot create the ItemReader [" + itemReader.getRef() + "]", e);
        }

        ItemProcessor itemProcessor = chunk.getProcessor();
        if (itemProcessor != null) {
            List<Property> itemProcessorProps = itemProcessor.getProperties() == null ? null : itemProcessor.getProperties().getPropertyList();
            try {

                InjectionReferences injectionRef = new InjectionReferences(runtimeWorkUnitExecution.getWorkUnitJobContext(), runtimeStepExecution, itemProcessorProps);

                processorProxy = ProxyFactory.createItemProcessorProxy(itemProcessor.getRef(), injectionRef, runtimeStepExecution);
                if (logger.isLoggable(Level.FINE)) {
                    logger.fine("Created ItemProcessorProxy for " + itemProcessor.getRef());
                }
            } catch (ArtifactValidationException e) {
                throw new BatchContainerServiceException("Cannot create the ItemProcessor [" + itemProcessor.getRef() + "]", e);
            }
        }

        ItemWriter itemWriter = chunk.getWriter();
        List<Property> itemWriterProps = itemWriter.getProperties() == null ? null : itemWriter.getProperties().getPropertyList();
        try {
            InjectionReferences injectionRef = new InjectionReferences(runtimeWorkUnitExecution.getWorkUnitJobContext(), runtimeStepExecution, itemWriterProps);

            writerProxy = ProxyFactory.createItemWriterProxy(itemWriter.getRef(), injectionRef, runtimeStepExecution);
            if (logger.isLoggable(Level.FINE)) {
                logger.fine("Created ItemWriterProxy for " + itemWriter.getRef());
            }
        } catch (ArtifactValidationException e) {
            throw new BatchContainerServiceException("Cannot create the ItemWriter [" + itemWriter.getRef() + "]", e);
        }

        InjectionReferences injectionRef = new InjectionReferences(runtimeWorkUnitExecution.getWorkUnitJobContext(), runtimeStepExecution, null);

        this.chunkListeners = runtimeWorkUnitExecution.getListenerFactory().getChunkListeners(getStep(), injectionRef, runtimeStepExecution);
        this.skipProcessListeners = runtimeWorkUnitExecution.getListenerFactory().getSkipProcessListeners(getStep(), injectionRef, runtimeStepExecution);
        this.skipReadListeners = runtimeWorkUnitExecution.getListenerFactory().getSkipReadListeners(getStep(), injectionRef, runtimeStepExecution);
        this.skipWriteListeners = runtimeWorkUnitExecution.getListenerFactory().getSkipWriteListeners(getStep(), injectionRef, runtimeStepExecution);
        this.retryProcessListeners = runtimeWorkUnitExecution.getListenerFactory().getRetryProcessListeners(getStep(), injectionRef, runtimeStepExecution);
        this.retryReadListeners = runtimeWorkUnitExecution.getListenerFactory().getRetryReadListeners(getStep(), injectionRef, runtimeStepExecution);
        this.retryWriteListeners = runtimeWorkUnitExecution.getListenerFactory().getRetryWriteListeners(getStep(), injectionRef, runtimeStepExecution);
        this.itemReadListeners = runtimeWorkUnitExecution.getListenerFactory().getItemReadListeners(getStep(), injectionRef, runtimeStepExecution);
        this.itemProcessListeners = runtimeWorkUnitExecution.getListenerFactory().getItemProcessListeners(getStep(), injectionRef, runtimeStepExecution);
        this.itemWriteListeners = runtimeWorkUnitExecution.getListenerFactory().getItemWriteListeners(getStep(), injectionRef, runtimeStepExecution);

        if (logger.isLoggable(Level.FINE)) {
            logger.fine("Setting contexts for chunk artifacts");
        }

        skipHandler = new SkipHandler(chunk, runtimeStepExecution);
        skipHandler.addSkipProcessListener(skipProcessListeners);
        skipHandler.addSkipReadListener(skipReadListeners);
        skipHandler.addSkipWriteListener(skipWriteListeners);

        retryHandler = new RetryHandler(chunk, runtimeStepExecution);

        retryHandler.addRetryProcessListener(retryProcessListeners);
        retryHandler.addRetryReadListener(retryReadListeners);
        retryHandler.addRetryWriteListener(retryWriteListeners);

        if (logger.isLoggable(Level.FINE))
            logger.exiting(sourceClass, sourceMethod);
    }
247653
def load_data_batch(self, data_batch):
        """"""
        _load_data(data_batch, self.data_arrays)
        _load_label(data_batch, self.label_arrays)
50379
@Override
    public void init(IBatchConfig batchConfig) throws BatchContainerServiceException {
        logger.entering(CLASSNAME, "init", batchConfig);
        this.batchConfig = batchConfig;
        logger.exiting(CLASSNAME, "init");
    }
1323142
private void init()
    {
        jdbcProperties = new Properties();
        dbcpProperties = new Properties();
        setFetchSize(0);
        this.setTestOnBorrow(true);
        this.setTestOnReturn(false);
        this.setTestWhileIdle(false);
        this.setLogAbandoned(false);
        this.setRemoveAbandoned(false);
    }
732409
protected void initialize() {

        CmsDriverManager driverManager = m_publishEngine.getDriverManager();
        CmsDbContext dbc = m_publishEngine.getDbContext(null);

        try {
            OpenCms.getMemoryMonitor().flushCache(CmsMemoryMonitor.CacheType.PUBLISH_HISTORY);
            // read all finished published jobs from the database
            List<CmsPublishJobInfoBean> publishJobs = driverManager.readPublishJobs(dbc, 1L, Long.MAX_VALUE);
            for (Iterator<CmsPublishJobInfoBean> i = publishJobs.iterator(); i.hasNext();) {
                CmsPublishJobInfoBean job = i.next();
                OpenCms.getMemoryMonitor().cachePublishJobInHistory(job);
            }
        } catch (CmsException exc) {
            dbc.rollback();
            if (LOG.isErrorEnabled()) {
                LOG.error(exc.getLocalizedMessage(), exc);
            }
        } finally {
            dbc.clear();
        }
    }
446651
@Override
  public void initialize() {
    if(databaseConnection == null) {
      return; // Supposedly we initialized already.
    }
    if(LOG.isDebugging()) {
      LOG.debugFine("Loading data from database connection.");
    }
    MultipleObjectsBundle bundle = databaseConnection.loadData();
    // Run at most once.
    databaseConnection = null;

    // Find DBIDs for bundle
    {
      DBIDs bids = bundle.getDBIDs();
      if(bids instanceof ArrayStaticDBIDs) {
        this.ids = (ArrayStaticDBIDs) bids;
      }
      else if(bids == null) {
        this.ids = DBIDUtil.generateStaticDBIDRange(bundle.dataLength());
      }
      else {
        this.ids = (ArrayStaticDBIDs) DBIDUtil.makeUnmodifiable(DBIDUtil.ensureArray(bids));
      }
    }
    // Replace id representation (it would be nicer if we would not need
    // DBIDView at all)
    this.idrep = new DBIDView(this.ids);
    relations.add(this.idrep);
    getHierarchy().add(this, idrep);

    DBIDArrayIter it = this.ids.iter();

    int numrel = bundle.metaLength();
    for(int i = 0; i < numrel; i++) {
      SimpleTypeInformation<?> meta = bundle.meta(i);
      @SuppressWarnings("unchecked")
      SimpleTypeInformation<Object> ometa = (SimpleTypeInformation<Object>) meta;
      WritableDataStore<Object> store = DataStoreUtil.makeStorage(ids, DataStoreFactory.HINT_DB, ometa.getRestrictionClass());
      for(it.seek(0); it.valid(); it.advance()) {
        store.put(it, bundle.data(it.getOffset(), i));
      }
      Relation<?> relation = new MaterializedRelation<>(ometa, ids, null, store);
      relations.add(relation);
      getHierarchy().add(this, relation);

      // Try to add indexes where appropriate
      for(IndexFactory<?> factory : indexFactories) {
        if(factory.getInputTypeRestriction().isAssignableFromType(ometa)) {
          @SuppressWarnings("unchecked")
          final IndexFactory<Object> ofact = (IndexFactory<Object>) factory;
          @SuppressWarnings("unchecked")
          final Relation<Object> orep = (Relation<Object>) relation;
          final Index index = ofact.instantiate(orep);
          Duration duration = LOG.isStatistics() ? LOG.newDuration(index.getClass().getName() + ".construction").begin() : null;
          index.initialize();
          if(duration != null) {
            LOG.statistics(duration.end());
          }
          getHierarchy().add(relation, index);
        }
      }
    }

    // fire insertion event
    eventManager.fireObjectsInserted(ids);
  }
1331444
protected void initBatchHandlers() {
    if (batchHandlers == null) {
      batchHandlers = new HashMap<String, BatchJobHandler<?>>();

      MigrationBatchJobHandler migrationHandler = new MigrationBatchJobHandler();
      batchHandlers.put(migrationHandler.getType(), migrationHandler);

      ModificationBatchJobHandler modificationHandler = new ModificationBatchJobHandler();
      batchHandlers.put(modificationHandler.getType(), modificationHandler);

      DeleteProcessInstancesJobHandler deleteProcessJobHandler = new DeleteProcessInstancesJobHandler();
      batchHandlers.put(deleteProcessJobHandler.getType(), deleteProcessJobHandler);

      DeleteHistoricProcessInstancesJobHandler deleteHistoricProcessInstancesJobHandler = new DeleteHistoricProcessInstancesJobHandler();
      batchHandlers.put(deleteHistoricProcessInstancesJobHandler.getType(), deleteHistoricProcessInstancesJobHandler);

      SetJobRetriesJobHandler setJobRetriesJobHandler = new SetJobRetriesJobHandler();
      batchHandlers.put(setJobRetriesJobHandler.getType(), setJobRetriesJobHandler);

      SetExternalTaskRetriesJobHandler setExternalTaskRetriesJobHandler = new SetExternalTaskRetriesJobHandler();
      batchHandlers.put(setExternalTaskRetriesJobHandler.getType(), setExternalTaskRetriesJobHandler);

      RestartProcessInstancesJobHandler restartProcessInstancesJobHandler = new RestartProcessInstancesJobHandler();
      batchHandlers.put(restartProcessInstancesJobHandler.getType(), restartProcessInstancesJobHandler);

      UpdateProcessInstancesSuspendStateJobHandler suspendProcessInstancesJobHandler = new UpdateProcessInstancesSuspendStateJobHandler();
      batchHandlers.put(suspendProcessInstancesJobHandler.getType(), suspendProcessInstancesJobHandler);

      DeleteHistoricDecisionInstancesJobHandler deleteHistoricDecisionInstancesJobHandler = new DeleteHistoricDecisionInstancesJobHandler();
      batchHandlers.put(deleteHistoricDecisionInstancesJobHandler.getType(), deleteHistoricDecisionInstancesJobHandler);

      ProcessSetRemovalTimeJobHandler processSetRemovalTimeJobHandler = new ProcessSetRemovalTimeJobHandler();
      batchHandlers.put(processSetRemovalTimeJobHandler.getType(), processSetRemovalTimeJobHandler);
    }

    if (customBatchJobHandlers != null) {
      for (BatchJobHandler<?> customBatchJobHandler : customBatchJobHandlers) {
        batchHandlers.put(customBatchJobHandler.getType(), customBatchJobHandler);
      }
    }
  }
1081069
private synchronized void initBuckets() {
    final SparseIntArray bucketSizes = mPoolParams.bucketSizes;

    // create the buckets
    if (bucketSizes != null) {
      fillBuckets(bucketSizes);
      mAllowNewBuckets = false;
    } else {
      mAllowNewBuckets = true;
    }
  }
81374
private static void initStreams() {
		Log.info("Initializing Streams...");
		MetadataCommand command = (MetadataCommand) PreConfigurationContext.getCommandByName("-metadata");
		BatchCommand batchCommand = (BatchCommand) PreConfigurationContext.getCommandByName("-batchSize");
		
		AsyncAnnotationAuditOutputStream asyncAnnotationStream = new AsyncAnnotationAuditOutputStream(
                new AuditProcessOutputStream(Context.getConfigContext()), annotationTransformer);
		
        AuditProcessOutputStream processStream =  new AuditProcessOutputStream(Context.getConfigContext());
        
        
		if (command.isAsync()) {
		    MetadataLookupStream metadataStream;
		    if(batchCommand.getBatchSize() > 0) {
	            BatchProcessStream batchStream = new BatchProcessStream(processStream, batchCommand.getBatchSize());
	            metadataStream = new MetadataLookupStream(batchStream);
		    } else {
		        metadataStream = new MetadataLookupStream(processStream);
		    }
			
			AsyncAuditOutputStream asyncStream = new AsyncAuditOutputStream(metadataStream, asyncAnnotationStream);
			auditStream = new AuditEventOutputStream(asyncStream, configContext);
		} else {
		    AsyncAuditOutputStream asyncStream;
		    if(batchCommand.getBatchSize() > 0) {
                BatchProcessStream batchStream = new BatchProcessStream(processStream, batchCommand.getBatchSize());
                asyncStream = new AsyncAuditOutputStream(batchStream, asyncAnnotationStream);
            } else {
                asyncStream = new AsyncAuditOutputStream(processStream, asyncAnnotationStream);
            }
			MetadataLookupStream metadataStream = new MetadataLookupStream(asyncStream);
			auditStream = new AuditEventOutputStream(metadataStream, configContext);
		}

		Log.info("Audit Streams Initialized.");
	}