Iterates through the inputs of the pipelinen and determines the
        all prerequisite pipelines
351982
def prerequisites(self):
        
        # Loop through the inputs to the pipeline and add the instancemethods
        # for the pipelines to generate each of the processed inputs
        prereqs = defaultdict(set)
        for input in self.inputs:  # @ReservedAssignment
            spec = self._study.spec(input)
            # Could be an input to the study or optional acquired spec
            if spec.is_spec and spec.derived:
                prereqs[spec.pipeline_getter].add(input.name)
        return prereqs
183423
def _get_input_steps(self):
        
        input_steps = []

        for step in self.steps_sorted:
            parent_steps = self._parent_steps(step)

            if len(parent_steps) == 0:
                input_steps.append(step)

        return input_steps
1244363
private void pumpInputData() throws IOException {
    for (Input input : inputs) {
      if (input.pump == null) {
        continue;
      }
      OutputStream os;
      if (input.pipeName.equals(STDIN_FILE_NAME)) {
        os = process.getOutputStream();
      } else {
        throw new IOException("Only stdin piping is supported so far.");
      }
      input.pump.pump(os);
    }
  }
1349991
def pipeline(steps, initial=None):
    

    def apply(result, step):
        return step(result)

    return reduce(apply, steps, initial)
749781
private void reconstructExecutionPipelines() {

		final Iterator<ExecutionStage> it = this.stages.iterator();
		while (it.hasNext()) {

			it.next().reconstructExecutionPipelines();
		}
	}
759331
def _find_output_dependencies(self, outputs):
        
        dependencies = []
        for address in outputs:
            dependencies.extend(
                self._predecessor_tree.find_write_predecessors(address))
        return dependencies
901895
def get_pipeline_names():
  """"""
  class_path_set = set()
  for cls in _PipelineMeta._all_classes:
      if cls.class_path is not None:
        class_path_set.add(cls.class_path)
  return sorted(class_path_set)
4469
def _parse_model(topology, scope, model, inputs=None, outputs=None):
    '''
    
    '''

    if inputs is None:
        inputs = list()
    if outputs is None:
        outputs = list()

    model_type = model.WhichOneof('Type')
    if model_type in ['pipeline', 'pipelineClassifier', 'pipelineRegressor']:
        _parse_pipeline_model(topology, scope, model, inputs, outputs)
    elif model_type in ['neuralNetworkClassifier', 'neuralNetworkRegressor', 'neuralNetwork']:
        _parse_neural_network_model(topology, scope, model, inputs, outputs)
    else:
        _parse_simple_model(topology, scope, model, inputs, outputs)
469634
function getInputSources(dirname, exclude, include, focusContractPath, callback) {
  let filesRead = 0;
  const sources = {};

  // get all file names
  dir.files(dirname, (filesError, files) => {
    if (filesError) {
      throwError(`while while getting input sources ${filesError}`);
    }

    // if no files in directory, then fire callback with empty sources
    if (files.length === 0) {
      callback(null, sources);
    } else {
      // read all files
      dir.readFiles(dirname, (readFilesError, content, filename, next) => {
        if (readFilesError) {
          throwError(`while getting input sources ${readFilesError}`);
        }

        // parsed filename
        const parsedDirName = dirname.replace('./', '');
        const parsedFileName = filename.replace(parsedDirName, '').replace(/^\//, '');
        const onlyFilename = filename.substr(filename.lastIndexOf('/') + 1);

        // add input sources to output
        if (filename.substr(filename.lastIndexOf('.') + 1) === 'sol'
          && filenameInclude(filename, exclude, include)
          && onlyFilename.charAt(0) !== '~'
          && onlyFilename.charAt(0) !== '#'
          && onlyFilename.charAt(0) !== '.') {
          sources[parsedFileName] = content;
        }

        // increase files readFiles
        filesRead += 1;

        // process next file
        if (filesRead === files.length) {
          // filter sources by focus dependencies
          if (focusContractPath) {
            callback(null, buildDependantsSourceTree(focusContractPath, sources));
          } else {
            callback(null, sources);
          }
        } else {
          next();
        }
      });
    }
  });
}
1222481
def _define_output_buffers(self):
        

        # First define buffers that need input data
        self.target_buffers = {
            None: [(step, self.buffers[step]) for step in self._get_input_steps()]
        }

        # Go through all steps and append the buffers of their child nodes
        for step in self.steps_sorted:
            if step != self:
                child_steps = [edge[1] for edge in self.graph.out_edges(step)]
                self.target_buffers[step] = [(child_step, self.buffers[child_step]) for child_step in child_steps]
1261866
def text2text_distill_iterator(source_txt_path, target_txt_path,
                               distill_txt_path):
  """"""
  for inputs, targets, dist_targets in zip(
      txt_line_iterator(source_txt_path), txt_line_iterator(target_txt_path),
      txt_line_iterator(distill_txt_path)):
    yield {"inputs": inputs, "targets": targets, "dist_targets": dist_targets}
361037
def run_pipelines(pipeline_id_pattern,
                  root_dir,
                  use_cache=True,
                  dirty=False,
                  force=False,
                  concurrency=1,
                  verbose_logs=True,
                  progress_cb=None,
                  slave=False):
    """"""
    with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency,
                                               thread_name_prefix='T') as executor:
        try:
            results = []
            pending_futures = set()
            done_futures = set()
            finished_futures = []
            progress_thread = None
            progress_queue = None
            status_manager = status_mgr(root_dir)

            if progress_cb is not None:
                progress_queue = Queue()
                progress_thread = threading.Thread(target=progress_report_handler, args=(progress_cb, progress_queue))
                progress_thread.start()

            all_specs = specs_to_execute(pipeline_id_pattern, root_dir, status_manager, force, dirty, results)

            while True:

                done = None
                if len(done_futures) > 0:
                    done = done_futures.pop()
                    finished_futures.append(done)
                    done = done.result()[0]

                try:
                    spec = all_specs.send(done)
                except StopIteration:
                    spec = None

                if spec is None:
                    # Wait for all runners to idle...
                    if len(done_futures) == 0:
                        if len(pending_futures) > 0:
                            done_futures, pending_futures = \
                                concurrent.futures.wait(pending_futures,
                                                        return_when=concurrent.futures.FIRST_COMPLETED)
                            continue
                        else:
                            break
                    else:
                        continue

                if len(spec.validation_errors) > 0:
                    results.append(
                        ExecutionResult(spec.pipeline_id,
                                        False,
                                        {},
                                        ['init'] + list(map(str, spec.validation_errors)))
                    )
                    continue

                if slave:
                    ps = status_manager.get(spec.pipeline_id)
                    ps.init(spec.pipeline_details,
                            spec.source_details,
                            spec.validation_errors,
                            spec.cache_hash)
                    eid = gen_execution_id()
                    if ps.queue_execution(eid, 'manual'):
                        success, stats, errors = \
                            execute_pipeline(spec, eid,
                                             use_cache=use_cache)
                        results.append(ExecutionResult(
                            spec.pipeline_id,
                            success,
                            stats,
                            errors
                        ))
                    else:
                        results.append(
                            ExecutionResult(spec.pipeline_id,
                                            False,
                                            None,
                                            ['Already Running'])
                        )

                else:
                    f = executor.submit(remote_execute_pipeline,
                                        spec,
                                        root_dir,
                                        use_cache,
                                        verbose_logs,
                                        progress_queue)
                    pending_futures.add(f)

            for f in finished_futures:
                ret = f.result()
                results.append(ExecutionResult(*ret))

        except KeyboardInterrupt:
            pass
        finally:
            if slave:
                finalize()

            if progress_thread is not None:
                progress_queue.put(None)
                progress_thread.join()

    return results
156275
def pipeline(names, steps):
    """"""
    steps, times = zip(*map(_maybe_timed, steps))
    fit_time = sum(times)
    if any(s is FIT_FAILURE for s in steps):
        fit_est = FIT_FAILURE
    else:
        fit_est = Pipeline(list(zip(names, steps)))
    return fit_est, fit_time
1364832
def run_steps(node: Node, pipeline: RenderingPipeline, **args):
    """"""
    for step in pipeline.steps:
        result = step(node, pipeline=pipeline, **args)
        if isinstance(result, dict):
            args = {**args, **result}
1203868
def process(self, pipeline):
        """"""
        output = []
        for entry in pipeline:
            key = list(entry.keys())[0]
            # an environment block can be repeated
            if key == "env":
                self.data.env_list[0].update(entry[key])
                self.logger.debug("Updating environment at level 0 with %s",
                                  self.data.env_list[0])
                continue

            # after validation it can't be anything else but a stage
            # and the title is inside the round brackets:
            stage = Stage(self, re.match(r"stage\((?P<title>.*)\)", key).group("title"))
            result = stage.process(entry[key])
            output += result['output']
            if not result['success']:
                return {'success': False, 'output': output}

        # logging the output of the cleanup shell when registered
        for line in self.cleanup():
            output.append(line)
            self.logger.info(" | %s", line)

        self.event.succeeded()
        return {'success': True, 'output': output}
1040031
function iteratePipeFile () {
    if (!isIterating()) { return undefined; }

    let data = iterateData;
    let file = data.collection[data.i];

    data.i++;

    if (data.i >= data.collection.length) {
        resetIterateData();
        pipeLastFile(file);
    } else {
        pipeFile(file, iteratePipeFile);
    }
}
44141
def _get_prelim_dependencies(command_template, all_templates):
    
    deps = []
    for input in command_template.input_parts:
        if '.' not in input.alias:
            continue
        for template in all_templates:
            for output in template.output_parts:
                if input.fuzzy_match(output):
                    deps.append(template)
                    break
    return list(set(deps))