Reads Instances from the iterable and puts them in the input_queue.
772073
def _queuer(instances: Iterable[Instance],
            input_queue: Queue,
            num_workers: int,
            num_epochs: Optional[int]) -> None:
    
    epoch = 0

    while num_epochs is None or epoch < num_epochs:
        epoch += 1
        for instance in instances:
            input_queue.put(instance)

    # Now put a None for each worker, since each needs to receive one
    # to know that it's done.
    for _ in range(num_workers):
        input_queue.put(None)
71150
def _count_fastq_reads(in_fastq, min_reads):
    
    with open(in_fastq) as in_handle:
        items = list(itertools.takewhile(lambda i : i <= min_reads,
                                         (i for i, _ in enumerate(FastqGeneralIterator(in_handle)))))
    return len(items)
477307
@SuppressWarnings("unchecked")
	protected void initInputReaders() throws Exception {
		final int numInputs = getNumTaskInputs();
		final MutableReader<?>[] inputReaders = new MutableReader[numInputs];

		int numGates = 0;

		for (int i = 0; i < numInputs; i++) {
			//  ---------------- create the input readers ---------------------
			// in case where a logical input unions multiple physical inputs, create a union reader
			final int groupSize = this.config.getGroupSize(i);
			numGates += groupSize;
			if (groupSize == 1) {
				// non-union case
				inputReaders[i] = new MutableRecordReader<IOReadableWritable>(this);
			} else if (groupSize > 1){
				// union case
				MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
				for (int j = 0; j < groupSize; ++j) {
					readers[j] = new MutableRecordReader<IOReadableWritable>(this);
				}
				inputReaders[i] = new MutableUnionRecordReader<IOReadableWritable>(readers);
			} else {
				throw new Exception("Illegal input group size in task configuration: " + groupSize);
			}
		}
		this.inputReaders = inputReaders;

		// final sanity check
		if (numGates != this.config.getNumInputs()) {
			throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
		}
	}
710678
def _get_instances(self, instance):
        

        return [
            x
            for x in self.tree.body
            if isinstance(x, instance)
        ]
1249342
def build_from_queue(cls, input_queue, replay_size, batch_size):
    
    return cls(
        lambda: input_queue.dequeue_many(batch_size),
        replay_size,
        batch_size=batch_size)
890843
def read(self):
        """"""
        queue_path = os.path.join(self.config_dir, 'queue')
        if os.path.exists(queue_path):
            queue_file = open(queue_path, 'rb')
            try:
                self.queue = pickle.load(queue_file)
            except Exception:
                print('Queue file corrupted, deleting old queue')
                os.remove(queue_path)
                self.queue = {}
            queue_file.close()
        else:
            self.queue = {}
1170286
def input_loop():
    ''''''
    while mpstate.status.exit != True:
        try:
            if mpstate.status.exit != True:
                line = input(mpstate.rl.prompt)
        except EOFError:
            mpstate.status.exit = True
            sys.exit(1)
        mpstate.input_queue.put(line)
1324428
private static List<Object> readList(StreamingInput input)
		throws IOException
	{
		input.next(Token.LIST_START);

		List<Object> result = new ArrayList<Object>();
		while(input.peek() != Token.LIST_END)
		{
			// Read the value
			Object value = readDynamic(input);
			result.add(value);
		}

		input.next(Token.LIST_END);

		return result;
	}
1098163
private void processNewlyAvailableInstances() {
		synchronized (globalLock) {
			Instance instance;

			while ((instance = newlyAvailableInstances.poll()) != null) {
				if (instance.hasResourcesAvailable()) {
					instancesWithAvailableResources.put(instance.getTaskManagerID(), instance);
				}
			}
		}
	}
114614
private void readObject(java.io.ObjectInputStream s)
        throws java.io.IOException, ClassNotFoundException {
        // Read in size, and any hidden stuff
        s.defaultReadObject();

        // Read in (and discard) array length
        s.readInt();

        queue = new Object[size];

        // Read in all elements.
        for (int i = 0; i < size; i++)
            queue[i] = s.readObject();

        // Elements are guaranteed to be in "proper order", but the
        // spec has never explained what that might be.
        heapify();
    }
219676
private function processInstances(): void
    {
        $preliminaryMap = [];
        foreach ($this->instances as $instance) {
            $preliminaryMap[get_class($instance)][] = $instance;
            foreach (class_parents($instance) as $parentClass) {
                $preliminaryMap[$parentClass][] = $instance;
            }
            foreach (class_implements($instance) as $parentClass) {
                $preliminaryMap[$parentClass][] = $instance;
            }
        }

        $nextIndex    = 0;
        $hashMap      = []; // hash  => index
        $settingsData = []; // index => instance data
        $getIndex     = function (SettingsInterface $instance) use (&$nextIndex, &$hashMap, &$settingsData): int {
            $hash = spl_object_hash($instance);
            if (array_key_exists($hash, $hashMap) === true) {
                $index = $hashMap[$hash];
            } else {
                $hashMap[$hash]           = $nextIndex;
                $settingsData[$nextIndex] = $instance->get($this->getApplicationData());
                $index                    = $nextIndex++;
            }

            return $index;
        };

        $settingsMap  = []; // class => index
        $ambiguousMap = []; // class => true
        foreach ($preliminaryMap as $class => $instanceList) {
            if (count($instanceList) === 1) {
                $selected = $instanceList[0];
            } else {
                $selected = $this->selectChildSettings($instanceList);
            }
            if ($selected !== null) {
                $settingsMap[$class] = $getIndex($selected);
            } else {
                $ambiguousMap[$class] = true;
            }
        }

        $this->settingsMap  = $settingsMap;
        $this->settingsData = $settingsData;
        $this->ambiguousMap = $ambiguousMap;
        $this->isProcessed  = true;
    }
1211854
def read(self, keyed=False, extended=False, cast=True, relations=False, limit=None):
        
        result = []
        rows = self.iter(keyed=keyed, extended=extended, cast=cast, relations=relations)
        for count, row in enumerate(rows, start=1):
            result.append(row)
            if count == limit:
                break
        return result
1034323
@SuppressWarnings("unchecked")
	private void initInputReaders() throws Exception {
		int numGates = 0;
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(0);
		numGates += groupSize;
		if (groupSize == 1) {
			// non-union case
			inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(
					getEnvironment().getInputGate(0),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			inputReader = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(getEnvironment().getAllInputGates()),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}
		
		this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
		@SuppressWarnings({ "rawtypes" })
		final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
		this.reader = (MutableObjectIterator<IT>)iter;

		// final sanity check
		if (numGates != this.config.getNumInputs()) {
			throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
		}
	}
1345897
def read(self, read_list, verbose=False, log=False, n_per_proc=None):
        ""
        ret = []
        self.prep_input(read_list)
        L = len(self.file_list)
        if L == 0:
            return ret

        logger.info("Beginning to run sparser.")
        output_file_list = []
        if log:
            log_name = 'sparser_run_%s.log' % _time_stamp()
            outbuf = open(log_name, 'wb')
        else:
            outbuf = None
        try:
            if self.n_proc == 1:
                for fpath in self.file_list:
                    outpath, _ = self.read_one(fpath, outbuf, verbose)
                    if outpath is not None:
                        output_file_list.append(outpath)
            else:
                if n_per_proc is None:
                    n_per_proc = max(1, min(1000, L//self.n_proc//2))
                pool = None
                try:
                    pool = Pool(self.n_proc)
                    if n_per_proc is not 1:
                        batches = [self.file_list[n*n_per_proc:(n+1)*n_per_proc]
                                   for n in range(L//n_per_proc + 1)]
                        out_lists_and_buffs = pool.map(self.read_some,
                                                       batches)
                    else:
                        out_files_and_buffs = pool.map(self.read_one,
                                                       self.file_list)
                        out_lists_and_buffs = [([out_files], buffs)
                                               for out_files, buffs
                                               in out_files_and_buffs]
                finally:
                    if pool is not None:
                        pool.close()
                        pool.join()
                for i, (out_list, buff) in enumerate(out_lists_and_buffs):
                    if out_list is not None:
                        output_file_list += out_list
                    if log:
                        outbuf.write(b'Log for producing output %d/%d.\n'
                                     % (i, len(out_lists_and_buffs)))
                        if buff is not None:
                            buff.seek(0)
                            outbuf.write(buff.read() + b'\n')
                        else:
                            outbuf.write(b'ERROR: no buffer was None. '
                                         b'No logs available.\n')
                        outbuf.flush()
        finally:
            if log:
                outbuf.close()
                if verbose:
                    logger.info("Sparser logs may be found at %s." %
                                log_name)
        ret = self.get_output(output_file_list)
        return ret
831730
@Override
	public Iterator<T> sample(Iterator<T> input) {
		return sampleInCoordinator(sampleInPartition(input));
	}
1271425
@Override
	public void read(final DataInput in) throws IOException {
		int size = in.readInt();
		this.map.clear();

		try {
			for (; size > 0; size--) {
				final K key = this.keyClass.newInstance();
				final V val = this.valueClass.newInstance();
				key.read(in);
				val.read(in);
				this.map.put(key, val);
			}
		} catch (final InstantiationException e) {
			throw new RuntimeException(e);
		} catch (final IllegalAccessException e) {
			throw new RuntimeException(e);
		}
	}
609450
def run(self):
        ''''''
        for line in iter(self._fd.readline, ''):
            self._queue.put(line)