Returns a dictionary containing default job submission information such as the *job_id* and
        task *branches* covered by the job.
171985
def job_data(cls, job_id=dummy_job_id, branches=None, **kwargs):
        
        return dict(job_id=job_id, branches=branches or [])
1366541
def _create_job_info(self, job_dir):
        
        meta = self._build_job_meta(job_dir)

        self.logger.debug("Create job: %s" % meta)

        job_record = JobRecord.from_json(meta)
        job_record.save()
6493
def collection_for_job(cls, job):
        
        return cls.collection(job_model_repr=job.get_model_repr(), identifier=getattr(job, '_cached_identifier', job.identifier.hget()))
1325117
def job_stats_enhanced(job_id):
    
    stats_dict = {}
    with os.popen('bjobs -o "jobid run_time cpu_used  queue slots  stat exit_code start_time estimated_start_time finish_time delimiter=\'|\'" -noheader ' + str(job_id)) as f:
        try:
            line = f.readline()
            cols = line.split('|')
            stats_dict['job_id'] = cols[0]
            if cols[1] != '-':
                stats_dict['wallclock'] = timedelta(
                    seconds=float(cols[1].split(' ')[0]))
            if cols[2] != '-':
                stats_dict['cpu'] = timedelta(
                    seconds=float(cols[2].split(' ')[0]))
            stats_dict['queue'] = cols[3]
            stats_dict['status'] = cols[5]
            stats_dict['exit_code'] = cols[6]
            stats_dict['start'] = cols[7]
            stats_dict['start_time'] = cols[8]
            if stats_dict['status'] in ['DONE', 'EXIT']:
                stats_dict['end'] = cols[9]

            steps = []
            stats_dict['steps'] = steps
        except:
            with os.popen('bhist -l ' + str(job_id)) as f:
                try:
                    output = f.readlines()
                    for line in output:
                        if "Done successfully" in line:
                            stats_dict['status'] = 'DONE'
                            return stats_dict
                        elif "Completed <exit>" in line:
                            stats_dict['status'] = 'EXIT'
                            return stats_dict
                        else:
                            stats_dict['status'] = 'UNKNOWN'
                except Exception as e:
                    print(e)
                    print('LSF: Error reading job stats')
                    stats_dict['status'] = 'UNKNOWN'
    return stats_dict
883273
def touch_control_outputs(self):
        
        task = self.task

        # create the parent directory
        self._outputs["submission"].parent.touch()

        # get all branch indexes and chunk them by tasks_per_job
        branch_chunks = list(iter_chunks(task.branch_map.keys(), task.tasks_per_job))

        # submission output
        if not self._outputs["submission"].exists():
            submission_data = self.submission_data.copy()
            # set dummy submission data
            submission_data.jobs.clear()
            for i, branches in enumerate(branch_chunks):
                job_num = i + 1
                submission_data.jobs[job_num] = self.submission_data_cls.job_data(branches=branches)
            self._outputs["submission"].dump(submission_data, formatter="json", indent=4)

        # status output
        if "status" in self._outputs and not self._outputs["status"].exists():
            status_data = self.status_data_cls()
            # set dummy status data
            for i, branches in enumerate(branch_chunks):
                job_num = i + 1
                status_data.jobs[job_num] = self.status_data_cls.job_data(
                    status=self.job_manager.FINISHED, code=0)
            self._outputs["status"].dump(status_data, formatter="json", indent=4)
915162
def info(job_name_or_id):
    
    try:
        experiment = ExperimentClient().get(normalize_job_name(job_name_or_id))
    except FloydException:
        experiment = ExperimentClient().get(job_name_or_id)

    task_instance_id = get_module_task_instance_id(experiment.task_instances)
    task_instance = TaskInstanceClient().get(task_instance_id) if task_instance_id else None
    normalized_job_name = normalize_job_name(experiment.name)
    table = [["Job name", normalized_job_name],
             ["Created", experiment.created_pretty],
             ["Status", experiment.state], ["Duration(s)", experiment.duration_rounded],
             ["Instance", experiment.instance_type_trimmed],
             ["Description", experiment.description],
             ["Metrics", format_metrics(experiment.latest_metrics)]]
    if task_instance and task_instance.mode in ['jupyter', 'serving']:
        table.append(["Mode", task_instance.mode])
        table.append(["Url", experiment.service_url])
    if experiment.tensorboard_url:
        table.append(["TensorBoard", experiment.tensorboard_url])
    floyd_logger.info(tabulate(table))
465974
def do_some_work(
        self,
        work_dict):
    

    label = "do_some_work"

    log.info(("task - {} - start "
              "work_dict={}")
             .format(label,
                     work_dict))

    ret_data = {
        "job_results": ("some response key={}").format(
                            str(uuid.uuid4()))
    }

    log.info(("task - {} - result={} done")
             .format(
                 ret_data,
                 label))

    return ret_data
682042
def submit_jobs(self, link, job_dict=None, job_archive=None, stream=sys.stdout):
        
        if link is None:
            return JobStatus.no_job
        if job_dict is None:
            job_keys = link.jobs.keys()
        else:
            job_keys = sorted(job_dict.keys())

        # copy & reverse the keys b/c we will be popping item off the back of
        # the list
        unsubmitted_jobs = job_keys
        unsubmitted_jobs.reverse()

        failed = False
        if unsubmitted_jobs:
            if stream != sys.stdout:
                sys.stdout.write('Submitting jobs (%i): ' %
                                 len(unsubmitted_jobs))
                sys.stdout.flush()
        while unsubmitted_jobs:
            status = get_lsf_status()
            njob_to_submit = min(self._max_jobs - status['NJOB'],
                                 self._jobs_per_cycle,
                                 len(unsubmitted_jobs))

            if self._dry_run:
                njob_to_submit = len(unsubmitted_jobs)

            for i in range(njob_to_submit):
                job_key = unsubmitted_jobs.pop()

                # job_details = job_dict[job_key]
                job_details = link.jobs[job_key]
                job_config = job_details.job_config
                if job_details.status == JobStatus.failed:
                    clean_job(job_details.logfile, {}, self._dry_run)
                    # clean_job(job_details.logfile,
                    #          job_details.outfiles, self.args['dry_run'])

                job_config['logfile'] = job_details.logfile
                new_job_details = self.dispatch_job(
                    link, job_key, job_archive, stream)
                if new_job_details.status == JobStatus.failed:
                    failed = True
                    clean_job(new_job_details.logfile,
                              new_job_details.outfiles, self._dry_run)
                link.jobs[job_key] = new_job_details

            if unsubmitted_jobs:
                if stream != sys.stdout:
                    sys.stdout.write('.')
                    sys.stdout.flush()
                stream.write('Sleeping %.0f seconds between submission cycles\n' %
                             self._time_per_cycle)
                time.sleep(self._time_per_cycle)

        if failed:
            return JobStatus.failed

        if stream != sys.stdout:
            sys.stdout.write('!\n')

        return JobStatus.done
1262883
def list(self, job_ids, print_array_jobs = False, print_dependencies = False, long = False, print_times = False, status=Status, names=None, ids_only=False):
    """"""
    # configuration for jobs
    fields = ("job-id", "grid-id", "queue", "status", "job-name")
    lengths = (6, 17, 11, 12, 16)
    dependency_length = 0

    if print_dependencies:
      fields += ("dependencies",)
      lengths += (25,)
      dependency_length = lengths[-1]

    if long:
      fields += ("submitted command",)
      lengths += (43,)

    format = "{:^%d}  " * len(lengths)
    format = format % lengths

    # if ids_only:
    #   self.lock()
    #   for job in self.get_jobs():
    #     print(job.unique, end=" ")
    #   self.unlock()
    #   return

    array_format = "{0:^%d}  {1:>%d}  {2:^%d}  {3:^%d}" % lengths[:4]
    delimiter = format.format(*['='*k for k in lengths])
    array_delimiter = array_format.format(*["-"*k for k in lengths[:4]])
    header = [fields[k].center(lengths[k]) for k in range(len(lengths))]

    # print header
    if not ids_only:
      print('  '.join(header))
      print(delimiter)

    self.lock()
    for job in self.get_jobs(job_ids):
      job.refresh()
      if job.status in status and (names is None or job.name in names):
        if ids_only:
          print(job.unique, end=" ")
        else:
          print(job.format(format, dependency_length))
        if print_times:
          print(times(job))

        if (not ids_only) and print_array_jobs and job.array:
          print(array_delimiter)
          for array_job in job.array:
            if array_job.status in status:
              print(array_job.format(array_format))
              if print_times:
                print(times(array_job))
          print(array_delimiter)

    self.unlock()
561448
def job_info(self, **kwargs):
        
        search = ";".join(["%s : %s"%(k, str(kwargs[k])) for k in kwargs])
        params = {"owner_wsid": self.userid, "owner_pw": self.password,
                "conditions": search, "includeSystem": False}
        r = self._send_request("GetJobs", params=params)
        results = []
        for n in minidom.parseString(r.text).getElementsByTagName("CJJob"):
            results.append({})
            for e in n.childNodes:
                if e.nodeType != e.TEXT_NODE:
                    results[-1][e.tagName] = e.firstChild.data
        return results
347428
def _init_jobs_table():
    """"""
    _jobs_table = sqlalchemy.Table(
        'jobs', _METADATA,
        sqlalchemy.Column('job_id', sqlalchemy.UnicodeText, primary_key=True),
        sqlalchemy.Column('job_type', sqlalchemy.UnicodeText),
        sqlalchemy.Column('status', sqlalchemy.UnicodeText, index=True),
        sqlalchemy.Column('data', sqlalchemy.UnicodeText),
        sqlalchemy.Column('error', sqlalchemy.UnicodeText),
        sqlalchemy.Column('requested_timestamp', sqlalchemy.DateTime),
        sqlalchemy.Column('finished_timestamp', sqlalchemy.DateTime),
        sqlalchemy.Column('sent_data', sqlalchemy.UnicodeText),
        # Callback URL:
        sqlalchemy.Column('result_url', sqlalchemy.UnicodeText),
        # CKAN API key:
        sqlalchemy.Column('api_key', sqlalchemy.UnicodeText),
        # Key to administer job:
        sqlalchemy.Column('job_key', sqlalchemy.UnicodeText),
        )
    return _jobs_table
1181845
def job_to_dict(job):
    """"""

    data = OrderedDict()
    data['id'] = job.id
    data['name'] = job.name
    data['func'] = job.func_ref
    data['args'] = job.args
    data['kwargs'] = job.kwargs

    data.update(trigger_to_dict(job.trigger))

    if not job.pending:
        data['misfire_grace_time'] = job.misfire_grace_time
        data['max_instances'] = job.max_instances
        data['next_run_time'] = None if job.next_run_time is None else job.next_run_time

    return data
965331
def make_job_details(self, row_idx):
        
        row = self._table[row_idx]
        job_details = JobDetails.create_from_row(row)
        job_details.get_file_paths(self._file_archive, self._table_id_array)
        self._cache[job_details.fullkey] = job_details
        return job_details
592868
def packing_job_ext_info(job_lsit_DO):
    
    ext_info = sqllite_agent.execute(ScrapydJobExtInfoSQLSet.SELECT_BY_ID, (job_lsit_DO.job_id,))
    if ext_info is None or len(ext_info) <= 0: return
    ext_info = ext_info[0]
    job_lsit_DO.args = ext_info[1]
    job_lsit_DO.priority = ext_info[2]
    job_lsit_DO.creation_time = ext_info[3]
    job_lsit_DO.logs_name = str_to_list(ext_info[4], ',')
    job_lsit_DO.logs_url = str_to_list(ext_info[5], ',')
356076
def __job_complete_dict(complete_status, manager, job_id):
    
    return_code = manager.return_code(job_id)
    if return_code == PULSAR_UNKNOWN_RETURN_CODE:
        return_code = None
    stdout_contents = manager.stdout_contents(job_id).decode("utf-8")
    stderr_contents = manager.stderr_contents(job_id).decode("utf-8")
    job_directory = manager.job_directory(job_id)
    as_dict = dict(
        job_id=job_id,
        complete="true",  # Is this still used or is it legacy.
        status=complete_status,
        returncode=return_code,
        stdout=stdout_contents,
        stderr=stderr_contents,
        working_directory=job_directory.working_directory(),
        metadata_directory=job_directory.metadata_directory(),
        working_directory_contents=job_directory.working_directory_contents(),
        metadata_directory_contents=job_directory.metadata_directory_contents(),
        outputs_directory_contents=job_directory.outputs_directory_contents(),
        system_properties=manager.system_properties(),
        pulsar_version=pulsar_version,
    )
    return as_dict
381408
def register_jobs(self, job_dict):
        """"""
        njobs = len(job_dict)
        sys.stdout.write("Registering %i total jobs: " % njobs)
        for i, job_details in enumerate(job_dict.values()):
            if i % 10 == 0:
                sys.stdout.write('.')
                sys.stdout.flush()
            self.register_job(job_details)
        sys.stdout.write('!\n')
1017518
def results( self ):
        """"""
        rs = []
        for k in self._results.keys():
            # filter out pending job ids, which can be anything except dicts
            ars = [ res for res in self._results[k] if isinstance(res, dict) ]
            rs.extend(ars)
        return rs