Download the input sample

    :param JobFunctionWrappingJob job: passed by Toil automatically
    :param tuple sample: Tuple containing (UUID,URL) of a sample
    :param Namespace inputs: Stores input arguments (see main)
768716
def download_sample(job, sample, inputs):
    
    uuid, url = sample
    job.fileStore.logToMaster('Downloading sample: {}'.format(uuid))
    # Download sample
    tar_id = job.addChildJobFn(download_url_job, url, s3_key_path=inputs.ssec, disk='30G').rv()
    # Create copy of inputs for each sample
    sample_inputs = argparse.Namespace(**vars(inputs))
    sample_inputs.uuid = uuid
    sample_inputs.cores = multiprocessing.cpu_count()
    # Call children and follow-on jobs
    job.addFollowOnJobFn(process_sample, sample_inputs, tar_id, cores=2, disk='60G')
153257
def start_batch(job, input_args):
    
    samples = parse_sra(input_args['sra'])
    # for analysis_id in samples:
    job.addChildJobFn(download_and_transfer_sample, input_args, samples, cores=1, disk='30')
137362
private void readJobInformation(JobConf jobConf, JobInfo jobInfo) throws Exception {
  
    /*
     * Convert the input strings to URL
     */
    URL jobConfFileUrl = new URL(this._jobConfFile);
    URL jobHistoryFileUrl = new URL (this._jobHistoryFile);
    
    /*
     * Read the Job Configuration from the jobConfFile url
     */  
    jobConf.addResource(jobConfFileUrl);
    
    /* 
     * Read JobHistoryFile and build job counters to evaluate diagnostic rules
     */
    if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
      DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.get(jobConf));
    } else if (jobHistoryFileUrl.getProtocol().equals("file")) {
      DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.getLocal(jobConf));
    } else {
      throw new Exception("Malformed URL. Protocol: "+jobHistoryFileUrl.getProtocol());
    }
  }
137675
def generate(self, job):
        

        options = job.get_options().copy()
        job_name = options.pop('name', None)
        job_account = options.pop('account', None)
        job_walltime = options.pop('walltime', None)
        job_mem_per_cpu = options.pop('mem_per_cpu', None)
        job_memory = options.pop('memory', None)
        job_working_directory = options.pop('working_directory', None)
        job_error_path = options.pop('error_path', None)
        job_output_path = options.pop('output_path', None)
        job_dependency = options.pop('depending', None)
        job_shell = options.pop('shell', '/bin/bash')
        job_custom_options = options.pop('__custom__', [])

        directives_lines = []

        if job_name is not None:
            directives_lines.append('--job-name=%s' % job_name)

        if job_account is not None:
            directives_lines.append('--account=%s' % job_account)

        if job_working_directory is not None:
            directives_lines.append('--workdir=%s' % job_working_directory)

        if job_error_path is not None:
            directives_lines.append('--error=%s' % job_error_path)

        if job_output_path is not None:
            directives_lines.append('--output=%s' % job_output_path)

        if job_walltime is not None:
            directives_lines.append('--time=%s' %
                                    strfdelta(job_walltime, '%H:%M:%S'))

        if job_mem_per_cpu is not None:
            directives_lines.append('--mem-per-cpu=%d' % job_mem_per_cpu)

        if job_memory is not None:
            directives_lines.append('--mem=%d' % job_memory)

        if job_dependency is not None:
            master = job_dependency['job']
            dependency_type = SlurmPrinter.get_dependency_type(
                job_dependency['dependency_type']
            )
            job_id = master.get_id() if isinstance(master, JobInterface) else master

            directives_lines.append(
                '--dependency=%s:%s' %
                (dependency_type, job_id)
            )

        for custom_option in job_custom_options:
            directives_lines.append(custom_option)

        directives = '\n'.join([
            '#SBATCH %s' % directive for directive in directives_lines
        ])

        commands = '\n'.join([
            '\n'.join(command_container.get_commands()) for command_container in job.get_commands()
        ])

        script = '#!%s\n' % job_shell
        script += SlurmPrinter.get_header()
        script += directives
        script += '\n\n'
        script += commands

        return script
96472
def parse_input_samples(job, inputs):
    
    job.fileStore.logToMaster('Parsing input samples and batching jobs')
    samples = []
    if inputs.config:
        with open(inputs.config, 'r') as f:
            for line in f.readlines():
                if not line.isspace():
                    sample = line.strip().split(',')
                    assert len(sample) == 2, 'Error: Config file is inappropriately formatted.'
                    samples.append(sample)
    job.addChildJobFn(map_job, download_sample, samples, inputs)
734721
def sample_loop(job, uuid_list, inputs):
  

  for uuid_rg in uuid_list:

    uuid_items = uuid_rg.split(',')
    uuid = uuid_items[0]
    rg_line = None
    if len(uuid_items) > 1:
        rg_line = uuid_items[1]

    job.addChildJobFn(static_dag, uuid, rg_line, inputs)
409654
def _execute(job, f, o=None):
    
    # Re-use the same buffer for output, we will read from it after each
    # iteration.
    out = ctypes.create_string_buffer(RS_JOB_BLOCKSIZE)
    while True:
        block = f.read(RS_JOB_BLOCKSIZE)
        buff = Buffer()
        # provide the data block via input buffer.
        buff.next_in = ctypes.c_char_p(block)
        buff.avail_in = ctypes.c_size_t(len(block))
        buff.eof_in = ctypes.c_int(not block)
        # Set up our buffer for output.
        buff.next_out = ctypes.cast(out, ctypes.c_char_p)
        buff.avail_out = ctypes.c_size_t(RS_JOB_BLOCKSIZE)
        r = _librsync.rs_job_iter(job, ctypes.byref(buff))
        if o:
            o.write(out.raw[:RS_JOB_BLOCKSIZE - buff.avail_out])
        if r == RS_DONE:
            break
        elif r != RS_BLOCKED:
            raise LibrsyncError(r)
        if buff.avail_in > 0:
            # There is data left in the input buffer, librsync did not consume
            # all of it. Rewind the file a bit so we include that data in our
            # next read. It would be better to simply tack data to the end of
            # this buffer, but that is very difficult in Python.
            f.seek(f.tell() - buff.avail_in)
    if o and callable(getattr(o, 'seek', None)):
        # As a matter of convenience, rewind the output file.
        o.seek(0)
    return o
469536
def download(self, job_id, files, **kwargs):
        
        check_jobid(job_id)

        if not files:
            return
        if isinstance(files, string_types):
            files = [files]
        directory = False
        recursive = kwargs.pop('recursive', True)

        if 'destination' in kwargs and 'directory' in kwargs:
            raise TypeError("Only use one of 'destination' or 'directory'")
        elif 'destination' in kwargs:
            destination = Path(kwargs.pop('destination'))
            if len(files) != 1:
                raise ValueError("'destination' specified but multiple files "
                                 "given; did you mean to use 'directory'?")
        elif 'directory' in kwargs:
            destination = Path(kwargs.pop('directory'))
            directory = True
        if kwargs:
            raise TypeError("Got unexpected keyword arguments")

        # Might raise JobNotFound
        status, target, result = self.status(job_id)

        scp_client = self.get_scp_client()
        for filename in files:
            logger.info("Downloading %s", target / filename)
            if directory:
                scp_client.get(str(target / filename),
                               str(destination / filename),
                               recursive=recursive)
            else:
                scp_client.get(str(target / filename),
                               str(destination),
                               recursive=recursive)
1128244
function copy_job(jobName, newJobName, modifyFunction, customParams, callback) {
      var _doArgs35 = doArgs(arguments, ['string', 'string', 'function', ['object', {}], 'function']);

      var _doArgs36 = _slicedToArray(_doArgs35, 5);

      jobName = _doArgs36[0];
      newJobName = _doArgs36[1];
      modifyFunction = _doArgs36[2];
      customParams = _doArgs36[3];
      callback = _doArgs36[4];


      var self = this;

      this.get_config_xml(jobName, customParams, function (error, data) {
        if (error) {
          callback(error, data);
          return;
        }

        // Modify the data
        data = modifyFunction(data);

        self.create_job(newJobName, data, customParams, callback);
      });
    }
1125287
def _update_job(self, target, args, kwargs):
        """"""
        target_path, options = get_function_path_and_options(target)

        assert isinstance(args, (tuple, list)) or args is None
        assert isinstance(kwargs, dict) or kwargs is None

        if options:
            self.update_options(**options)

        self._options['job'] = (target_path, args, kwargs)
476073
def parse_config_file(job, ids, input_args):
    
    samples = []
    config = input_args['config']
    with open(config, 'r') as f:
        for line in f.readlines():
            if not line.isspace():
                sample = line.strip().split(',')
                samples.append(sample)
    for sample in samples:
        job.addChildJobFn(download_sample, ids, input_args, sample)
1004267
function (jobName, customParams, callback) {
      [jobName, customParams, callback] = doArgs(arguments, ['string', ['object', {}], 'function']);

      doRequest({
        urlPattern: [JOB_CONFIG, jobName],
        noparse: true
      }, customParams, function (error, data) {
        // Get only the XML response body
        if (error) {
          callback(error, data);
        } else {
          callback(null, data.body);
        }
      });
    }
1264817
function downloadFiles(job, iFile, cb) {
   if (isUndefined(cb)) {
      cb = iFile;
      iFile = 0;
   }

   if (iFile === job.document.files.length) {
      cb();
      return;
   } 

   var file = job.document.files[iFile];
   var filePath = path.resolve(job.document.workingDirectory || '', file.fileName);

   if (isUndefined(file.fileSource) || isUndefined(file.fileSource.url)) {
      job.inProgress({ operation: job.operation, step: 'download error, rollback pending', fileName: file.fileName }, function(err) {
         showJobsError(err);
         cb(new Error('fileSource url missing'));
      });
      return;
   }

   job.inProgress({ step: 'downloading', fileName: file.fileName }, function(err) {
      showJobsError(err);
      downloadFile(file.fileSource.url, filePath, function(downloadError) {
         if (isUndefined(downloadError)) {
            validateChecksum(filePath, file.checksum, function(checksumError) {
               if (isUndefined(checksumError)) {
                  validateSignature(filePath, file.signature, function(signatureError) {
                     if (isUndefined(signatureError)) {
                        downloadFiles(job, iFile + 1, cb);
                     } else {
                        cb(signatureError);
                     }
                  });
               } else {
                  cb(checksumError);
               }
            });
         } else {
            cb(downloadError);
         }
      });
   });
}
30230
def download_data(job, master_ip, inputs, known_snps, bam, hdfs_snps, hdfs_bam):
    

    log.info("Downloading known sites file %s to %s.", known_snps, hdfs_snps)
    call_conductor(job, master_ip, known_snps, hdfs_snps, memory=inputs.memory)

    log.info("Downloading input BAM %s to %s.", bam, hdfs_bam)
    call_conductor(job, master_ip, bam, hdfs_bam, memory=inputs.memory)
682042
def submit_jobs(self, link, job_dict=None, job_archive=None, stream=sys.stdout):
        
        if link is None:
            return JobStatus.no_job
        if job_dict is None:
            job_keys = link.jobs.keys()
        else:
            job_keys = sorted(job_dict.keys())

        # copy & reverse the keys b/c we will be popping item off the back of
        # the list
        unsubmitted_jobs = job_keys
        unsubmitted_jobs.reverse()

        failed = False
        if unsubmitted_jobs:
            if stream != sys.stdout:
                sys.stdout.write('Submitting jobs (%i): ' %
                                 len(unsubmitted_jobs))
                sys.stdout.flush()
        while unsubmitted_jobs:
            status = get_lsf_status()
            njob_to_submit = min(self._max_jobs - status['NJOB'],
                                 self._jobs_per_cycle,
                                 len(unsubmitted_jobs))

            if self._dry_run:
                njob_to_submit = len(unsubmitted_jobs)

            for i in range(njob_to_submit):
                job_key = unsubmitted_jobs.pop()

                # job_details = job_dict[job_key]
                job_details = link.jobs[job_key]
                job_config = job_details.job_config
                if job_details.status == JobStatus.failed:
                    clean_job(job_details.logfile, {}, self._dry_run)
                    # clean_job(job_details.logfile,
                    #          job_details.outfiles, self.args['dry_run'])

                job_config['logfile'] = job_details.logfile
                new_job_details = self.dispatch_job(
                    link, job_key, job_archive, stream)
                if new_job_details.status == JobStatus.failed:
                    failed = True
                    clean_job(new_job_details.logfile,
                              new_job_details.outfiles, self._dry_run)
                link.jobs[job_key] = new_job_details

            if unsubmitted_jobs:
                if stream != sys.stdout:
                    sys.stdout.write('.')
                    sys.stdout.flush()
                stream.write('Sleeping %.0f seconds between submission cycles\n' %
                             self._time_per_cycle)
                time.sleep(self._time_per_cycle)

        if failed:
            return JobStatus.failed

        if stream != sys.stdout:
            sys.stdout.write('!\n')

        return JobStatus.done
347313
def pass_job(db: JobDB, result_queue: Queue, always_cache=False):
    
    @pull
    def pass_job_stream(job_source):
        """Pull stream instance created by `pass_job`."""
        result_sink = result_queue.sink()

        for message in job_source():
            if message is EndOfQueue:
                return

            key, job = message
            if always_cache or ('store' in job.hints):
                status, retrieved_result = db.add_job_to_db(key, job)

                if status == 'retrieved':
                    result_sink.send(retrieved_result)
                    continue

                elif status == 'attached':
                    continue

            yield message

    return pass_job_stream
792020
public static File getPathtoProject(final File filename, final File traceFilename, final File inputMap, final Job job) {
        if (job.getGeneratecopyouter() != Job.Generate.OLDSOLUTION) {
            if (isOutFile(traceFilename, inputMap)) {
                return toFile(getRelativePathFromOut(traceFilename.getAbsoluteFile(), job));
            } else {
                return getRelativePath(traceFilename.getAbsoluteFile(), inputMap.getAbsoluteFile()).getParentFile();
            }
        } else {
            return FileUtils.getRelativePath(filename);
        }
    }