Given a :class:`~clusterjob.JobScript` instance, return a list of
        lines that encode the resource requirements, to be added at the top of
        the rendered job script
172133
def resource_headers(self, jobscript):
        
        resources = jobscript.resources
        lines = []
        cores_per_node = 1
        nodes = 1
        if 'ppn' in resources:
            cores_per_node *= resources['ppn']
        if 'threads' in resources:
            cores_per_node *= resources['threads']
        if 'nodes' in resources:
            nodes = resources['nodes']
        if len(set(['ppn', 'threads', 'nodes']).intersection(resources)) > 0:
            line = '%s -n %d' % (self.prefix, nodes*cores_per_node)
            if cores_per_node > 1:
                line += ' -R "span[ptiles=%d]"' % cores_per_node
            lines.append(line)
        for (key, val) in resources.items():
            if key in ['nodes', 'ppn', 'threads']:
                continue
            if key in self.resource_replacements:
                lsf_key = self.resource_replacements[key]
                if key == 'time':
                    val = time_to_minutes(val)
            else:
                lsf_key = key
            if val is None:
                continue
            if type(val) is bool:
                if val:
                    lines.append("%s %s" % (self.prefix, lsf_key))
            else:
                lines.append("%s %s %s" % (self.prefix, lsf_key, str(val)))
        return lines
540870
def export_capacities(self):
        
        return [export for cls in getmro(type(self)) if hasattr(cls, "EXPORT_TO") for export in cls.EXPORT_TO]
1323249
def write_requirements(reqs_linenum, req_file):
    """"""
    with open(req_file, 'r') as input:
        lines = input.readlines()

    for req in reqs_linenum:
        line_num = int(req[1])

        if hasattr(req[0], 'link'):
            lines[line_num - 1] = '{}\n'.format(req[0].link)
        else:
            lines[line_num - 1] = '{}\n'.format(req[0])

    with open(req_file, 'w') as output:
        output.writelines(lines)
1230931
def build_job_scripts(root_dir, jobs):
    
    # Loop over the job list, creating each job script in turn, and then adding
    # scriptPath to the Job object
    for job in jobs:
        scriptpath = os.path.join(root_dir, "jobs", job.name)
        with open(scriptpath, "w") as scriptfile:
            scriptfile.write("#!/bin/sh\n#$ -S /bin/bash\n%s\n" % job.script)
        job.scriptpath = scriptpath
207457
def _get_scripts_resource(pe):
    """"""
    res = None
    for entry in pe.DIRECTORY_ENTRY_RESOURCE.entries:
        if entry.name and entry.name.string == b"PYTHONSCRIPT":
            res = entry.directory.entries[0].directory.entries[0]
            break
    return res
453167
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])

        for comp in components:
            zcut = "zmax%i" % comp.zmax

            mktimelist = copy.copy(comp.mktimefilters)
            if not mktimelist:
                mktimelist.append('none')
            evtclasslist_keys = copy.copy(comp.evtclasses)
            if not evtclasslist_keys:
                evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]
            else:
                evtclasslist_vals = copy.copy(evtclasslist_keys)

            for mktimekey in mktimelist:
                for evtclassval in evtclasslist_vals:
                    fullkey = comp.make_key(
                        '%s_%s_{ebin_name}_%s_{evtype_name}' %
                        (evtclassval, zcut, mktimekey))
                    name_keys = dict(zcut=zcut,
                                     ebin=comp.ebin_name,
                                     psftype=comp.evtype_name,
                                     coordsys=comp.coordsys,
                                     irf_ver=NAME_FACTORY.irf_ver(),
                                     mktime=mktimekey,
                                     evclass=evtclassval,
                                     fullpath=True)

                    outfile = NAME_FACTORY.bexpcube(**name_keys)
                    cmap = NAME_FACTORY.ccube(**name_keys)
                    infile = NAME_FACTORY.ltcube(**name_keys)
                    logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                    job_configs[fullkey] = dict(cmap=cmap,
                                                infile=infile,
                                                outfile=outfile,
                                                irfs=NAME_FACTORY.irfs(**name_keys),
                                                hpx_order=min(
                                                    comp.hpx_order, args['hpx_order_max']),
                                                evtype=comp.evtype,
                                                logfile=logfile)

        return job_configs
1028225
def build_job_configs(self, args):
        
        job_configs = {}

        ttype = args['ttype']
        (targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
        if targets_yaml is None:
            return job_configs

        targets = load_yaml(targets_yaml)

        for target_name, target_list in targets.items():
            for targ_prof in target_list:
                name_keys = dict(target_type=ttype,
                                 target_name=target_name,
                                 profile=targ_prof,
                                 fullpath=True)
                targ_key = "%s_%s" % (target_name, targ_prof)
                input_path = NAME_FACTORY.sedfile(**name_keys)
                output_path = input_path.replace('.fits', '.png')
                logfile = make_nfs_path(input_path.replace('.fits', '.log'))
                job_config = dict(infile=input_path,
                                  outfile=output_path,
                                  logfile=logfile)
                job_configs[targ_key] = job_config

        return job_configs
365079
def yaml_to_job_list(yaml_file, base_dir):
    """"""
    yaml_dict = get_yaml_dict(yaml_file)
    job_list = []
    for key, value in yaml_dict.items():
        job_list.append((value, rel_to_cwd(base_dir, key)))

    return job_list
171985
def job_data(cls, job_id=dummy_job_id, branches=None, **kwargs):
        
        return dict(job_id=job_id, branches=branches or [])
765552
def getJobIDsForResultsFile(self, resultsFile):
        
        jobIDs = []
        for line in self._runParasol(['-extended', 'list', 'jobs'])[1]:
            fields = line.strip().split()
            if len(fields) == 0 or fields[-1] != resultsFile:
                continue
            jobID = fields[0]
            jobIDs.append(int(jobID))
        return set(jobIDs)
399946
def compile_jobgroups_from_joblist(joblist, jgprefix, sgegroupsize):
    """"""
    jobcmds = defaultdict(list)
    for job in joblist:
        jobcmds[job.command.split(' ', 1)[0]].append(job.command)
    jobgroups = []
    for cmds in list(jobcmds.items()):
        # Break arglist up into batches of sgegroupsize (default: 10,000)
        sublists = split_seq(cmds[1], sgegroupsize)
        count = 0
        for sublist in sublists:
            count += 1
            sge_jobcmdlist = ['\"%s\"' % jc for jc in sublist]
            jobgroups.append(JobGroup("%s_%d" % (jgprefix, count),
                                      "$cmds",
                                      arguments={'cmds': sge_jobcmdlist}))
    return jobgroups
844055
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        if self._comp_dict is None or self._comp_dict_file != args['library']:
            self._comp_dict_file = args['library']
            self._comp_dict = make_catalog_comp_dict(sources=self._comp_dict_file,
                                                     basedir=NAME_FACTORY.base_dict['basedir'])
        else:
            print ("Using cached catalog dict from %s" % args['library'])

        catalog_info_dict = self._comp_dict['catalog_info_dict']
        comp_info_dict = self._comp_dict['comp_info_dict']

        n_src_per_job = args['nsrc']

        if args['make_xml']:
            SrcmapsCatalog_SG._make_xml_files(catalog_info_dict, comp_info_dict)

        for catalog_name, catalog_info in catalog_info_dict.items():

            n_cat_src = len(catalog_info.catalog.table)
            n_job = int(math.ceil(float(n_cat_src) / n_src_per_job))

            for comp in components:
                zcut = "zmax%i" % comp.zmax
                key = comp.make_key('{ebin_name}_{evtype_name}')
                name_keys = dict(zcut=zcut,
                                 sourcekey=catalog_name,
                                 ebin=comp.ebin_name,
                                 psftype=comp.evtype_name,
                                 coordsys=comp.coordsys,
                                 irf_ver=NAME_FACTORY.irf_ver(),
                                 mktime='none',
                                 fullpath=True)

                for i_job in range(n_job):
                    full_key = "%s_%02i" % (key, i_job)
                    srcmin = i_job * n_src_per_job
                    srcmax = min(srcmin + n_src_per_job, n_cat_src)
                    outfile = NAME_FACTORY.srcmaps(
                        **name_keys).replace('.fits', "_%02i.fits" % (i_job))
                    logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                    job_configs[full_key] = dict(cmap=NAME_FACTORY.ccube(**name_keys),
                                                 expcube=NAME_FACTORY.ltcube(**name_keys),
                                                 irfs=NAME_FACTORY.irfs(**name_keys),
                                                 bexpmap=NAME_FACTORY.bexpcube(**name_keys),
                                                 outfile=outfile,
                                                 logfile=logfile,
                                                 srcmdl=catalog_info.srcmdl_name,
                                                 evtype=comp.evtype,
                                                 srcmin=srcmin,
                                                 srcmax=srcmax)

        return job_configs
650565
def generate_script(self):
        
        self.script = ""        # Holds the script string
        total = 1               # total number of jobs in this group

        # for now, SGE_TASK_ID becomes TASK_ID, but we base it at zero
        self.script += """let "TASK_ID=$SGE_TASK_ID - 1"\n"""

        # build the array definitions; force ordering for Python3.5 tests
        for key in sorted(self.arguments.keys()):
            values = self.arguments[key]
            line = ("%s_ARRAY=( " % (key))
            for value in values:
                line += value
                line += " "
            line += " )\n"
            self.script += line
            total *= len(values)
        self.script += "\n"

        # now, build the decoding logic in the script; force ordering
        for key in sorted(self.arguments.keys()):
            count = len(self.arguments[key])
            self.script += """let "%s_INDEX=$TASK_ID %% %d"\n""" % (key, count)
            self.script += """%s=${%s_ARRAY[$%s_INDEX]}\n""" % (key, key, key)
            self.script += """let "TASK_ID=$TASK_ID / %d"\n""" % (count)

        # now, add the command to run the job
        self.script += "\n"
        self.script += self.command
        self.script += "\n"

        # set the number of tasks in this group
        self.tasks = total
13779
def get_requirements(*args):
    """"""
    requirements = set()
    contents = get_contents(*args)
    for line in contents.splitlines():
        # Strip comments.
        line = re.sub(r'^#.*|\s#.*', '', line)
        # Ignore empty lines
        if line and not line.isspace():
            requirements.add(re.sub(r'\s+', '', line))
    return sorted(requirements)
568569
def build_job_configs(self, args):
        
        job_configs = {}

        comp_file = args.get('comp', None)
        if comp_file is not None:
            comp_dict = yaml.safe_load(open(comp_file))
            coordsys = comp_dict.pop('coordsys')
            for v in comp_dict.values():
                v['coordsys'] = coordsys
        else:
            return job_configs

        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])

        inputfiles = create_inputlist(args['ft1file'])
        outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes')
        data_ver = NAME_FACTORY.base_dict['data_ver']

        for idx, infile in enumerate(inputfiles):
            key = "%06i" % idx
            key_scfile = "%03i" % (idx + 1)
            output_dir = os.path.join(outdir_base, key)
            try:
                os.mkdir(output_dir)
            except OSError:
                pass
            scfile = args['ft2file'].replace('.lst', '_%s.fits' % key_scfile)
            logfile = make_nfs_path(os.path.join(output_dir,
                                                 'scatter_mk_%s_%s.log' % (data_ver, key)))

            job_configs[key] = comp_dict.copy()
            job_configs[key].update(dict(ft1file=infile,
                                         scfile=scfile,
                                         comp=args['comp'],
                                         hpx_order_max=args['hpx_order_max'],
                                         outdir=outdir_base,
                                         outkey=key,
                                         logfile=logfile,
                                         pfiles=output_dir))

        return job_configs
270961
def copy_job_order(job, job_order_object):
    # type: (Any, Any) -> Any
    """"""
    if not hasattr(job, "tool"):
        # direct command line tool execution
        return job_order_object
    customised_job = {}  # new job object for RO
    for each, i in enumerate(job.tool["inputs"]):
        with SourceLine(job.tool["inputs"], each, WorkflowException,
                        _logger.isEnabledFor(logging.DEBUG)):
            iid = shortname(i["id"])
            if iid in job_order_object:
                customised_job[iid] = copy.deepcopy(job_order_object[iid])
                # add the input element in dictionary for provenance
            elif "default" in i:
                customised_job[iid] = copy.deepcopy(i["default"])
                # add the default elements in the dictionary for provenance
            else:
                pass
    return customised_job
374779
def _format_job_instance(job):
    '''
    
    '''
    ret = {'Function': job.get('fun', 'unknown-function'),
           'Arguments': salt.utils.json.loads(job.get('arg', '[]')),
           # unlikely but safeguard from invalid returns
           'Target': job.get('tgt', 'unknown-target'),
           'Target-type': job.get('tgt_type', 'list'),
           'User': job.get('user', 'root')}
    # TODO: Add Metadata support when it is merged from develop
    return ret