Write the submission script. Return (script, num_flows_in_batch)
2419
def _get_script_nflows(self):
        
        flows_torun = [f for f in self.flows if not f.all_ok]
        if not flows_torun:
            return "", 0

        executable = [
            'export _LOG=%s' % self.log_file.path,
            'date1=$(date +"%s")',
            'echo Running abirun.py in batch mode > ${_LOG}',
            " ",
        ]
        app = executable.append

        # Build list of abirun commands and save the name of the log files.
        self.sched_logs, num_flows = [], len(flows_torun)
        for i, flow in enumerate(flows_torun):

            logfile = os.path.join(self.workdir, "log_" + os.path.basename(flow.workdir))

            app("echo Starting flow %d/%d on: `date` >> ${LOG}" % (i+1, num_flows))
            app("\nabirun.py %s scheduler > %s" % (flow.workdir, logfile))
            app("echo Returning from abirun on `date` with retcode $? >> ${_LOG}")

            assert logfile not in self.sched_logs
            self.sched_logs.append(logfile)

        # Remove the batch pid_file and compute elapsed time.
        executable.extend([
            " ",
            "# Remove batch pid file",
            'rm %s' % self.batch_pidfile.path,
            " ",
            "# Compute elapsed time",
            'date2=$(date +"%s")',
            'diff=$(($date2-$date1))',
            'echo $(($diff / 60)) minutes and $(($diff % 60)) seconds elapsed. >> ${_LOG}'
        ])

        return self.qadapter.get_script_str(
            job_name=self.name,
            launch_dir=self.workdir,
            executable=executable,
            qout_path=self.qout_file.path,
            qerr_path=self.qerr_file.path,
        ), num_flows
688319
def write(self):
        
        # write data
        self.project_manager.writeInput(session=self.db_session,
                                        directory=self.project_directory,
                                        name=self.project_manager.name)
1039963
def add_flow(self, flow):
        
        from .flows import Flow
        flow = Flow.as_flow(flow)

        if flow in self.flows:
            raise self.Error("Cannot add same flow twice!")

        if not flow.allocated:
            # Set the workdir of the flow here. Create a dir in self.workdir with name flow.name
            flow_workdir = os.path.join(self.workdir, os.path.basename(flow.name))
            if flow_workdir in (flow.workdir for flow in self.flows):
                raise self.Error("Two flows have the same name and hence the same workdir!")
            flow.allocate(workdir=flow_workdir)

        # Check if we are already using a scheduler to run this flow
        flow.check_pid_file()
        flow.set_spectator_mode(False)

        flow.check_status(show=False)

        #if flow.all_ok:
        #    print("flow.all_ok: Ignoring %s" % flow)
        #    return 0

        self.flows.append(flow)
        #print("Flow %s added to the BatchLauncher" % flow)

        return 1
1038979
def write(self):
        '''
        '''
        if not self.output_files.valid():
            raise ValueError(
                f'Cannot write signature with undetermined output {self.output_files}'
            )
        else:
            if 'TARGET' in env.config['SOS_DEBUG'] or 'ALL' in env.config['SOS_DEBUG']:
                env.log_to_file(
                    'TARGET',
                    f'write signature {self.sig_id} with output {self.output_files}'
                )
        ret = super(RuntimeInfo, self).write()
        if ret is False:
            env.logger.debug(f'Failed to write signature {self.sig_id}')
            return ret
        send_message_to_controller(['step_sig', self.sig_id, ret])
        send_message_to_controller([
            'workflow_sig', 'tracked_files', self.sig_id,
            repr({
                'input_files': [
                    str(f.resolve())
                    for f in self.input_files
                    if isinstance(f, file_target)
                ],
                'dependent_files': [
                    str(f.resolve())
                    for f in self.dependent_files
                    if isinstance(f, file_target)
                ],
                'output_files': [
                    str(f.resolve())
                    for f in self.output_files
                    if isinstance(f, file_target)
                ]
            })
        ])
        return True
708228
def _submit_and_wait(cmd, cores, config, output_dir):
    
    batch_script = "submit_bcl2fastq.sh"
    if not os.path.exists(batch_script + ".finished"):
        if os.path.exists(batch_script + ".failed"):
            os.remove(batch_script + ".failed")
        with open(batch_script, "w") as out_handle:
            out_handle.write(config["process"]["bcl2fastq_batch"].format(
                cores=cores, bcl2fastq_cmd=" ".join(cmd), batch_script=batch_script))
        submit_cmd = utils.get_in(config, ("process", "submit_cmd"))
        subprocess.check_call(submit_cmd.format(batch_script=batch_script), shell=True)
        # wait until finished or failure checkpoint file
        while 1:
            if os.path.exists(batch_script + ".finished"):
                break
            if os.path.exists(batch_script + ".failed"):
                raise ValueError("bcl2fastq batch script failed: %s" %
                                 os.path.join(output_dir, batch_script))
            time.sleep(5)
27683
def submit(self, job):
        
        script = self._printer.generate(job)
        stdin, stdout, stderr = self._ssh.exec_command('sbatch')
        stdin.write(script)
        stdin.flush()
        stdin.channel.shutdown_write()
        return stdout.read()
1299192
def WriteFlowRequests(self, requests):
    """"""
    flow_processing_requests = []

    for request in requests:
      if (request.client_id, request.flow_id) not in self.flows:
        raise db.AtLeastOneUnknownFlowError([(request.client_id,
                                              request.flow_id)])

    for request in requests:
      key = (request.client_id, request.flow_id)
      request_dict = self.flow_requests.setdefault(key, {})
      request_dict[request.request_id] = request.Copy()
      request_dict[request.request_id].timestamp = rdfvalue.RDFDatetime.Now()

      if request.needs_processing:
        flow = self.flows[(request.client_id, request.flow_id)]
        if flow.next_request_to_process == request.request_id:
          flow_processing_requests.append(
              rdf_flows.FlowProcessingRequest(
                  client_id=request.client_id,
                  flow_id=request.flow_id,
                  delivery_time=request.start_time))

    if flow_processing_requests:
      self.WriteFlowProcessingRequests(flow_processing_requests)
132640
def submit(self, job):
        
        from subprocess import Popen, PIPE
        script = self._printer.generate(job)
        process = Popen('sbatch', stdout=PIPE, stdin=PIPE, stderr=PIPE)
        stdout, sterr = process.communicate(input=script)
        process.stdin.close()
655537
def _write(self, data):
        """"""
        stream = yaml.dump(data, default_flow_style=False)
        self.path.write_text(stream)
963565
def streamWrite(self, size):
        
        size = int(size)
        if size == 0:
            self._close()
            return

        self._sendResult(dict(message="writing...", stream=True, size=size))
        data = sys.stdin.read(size)
        self.stream.write(data)
840639
def _process_submission(self, submission: praw.models.Submission):
        
        self._func_submission(submission, *self._func_submission_args)
435463
def pbs_submit(script, queue, headers, resources, envvars)
        attribs = []
        headers.each do |name, value|
          attribs << { name: name, value: value }
        end
        resources.each do |rsc, value|
          attribs << { name: :Resource_List, resource: rsc, value: value }
        end
        unless envvars.empty?
          attribs << {
            name: :Variable_List,
            value: envvars.map {|k,v| "#{k}=#{v}"}.join(",")
          }
        end

        connect do |cid|
          attropl = Torque::Attropl.from_list attribs
          Torque.pbs_submit cid, attropl, script, queue, nil
        end
      end
1023300
def emit(self, batch):
        
        try:
            self.client.submitBatches([batch])
            # it will call http_transport.flush() and
            # status code and message will be updated
            code = self.http_transport.code
            msg = self.http_transport.message
            if code >= 300 or code < 200:
                logging.error("Traces cannot be uploaded;\
                        HTTP status code: {}, message {}".format(code, msg))
        except Exception as e:  # pragma: NO COVER
            logging.error(getattr(e, 'message', e))

        finally:
            if self.http_transport.isOpen():
                self.http_transport.close()
651493
def _submit_to_queue(self, script_file):
        """"""
        if sys.version_info[0] < 3:
            process = Popen(['qsub', script_file], stdout=PIPE, stderr=PIPE)
        else:
            # need string not bytes so must use universal_newlines
            process = Popen(['qsub', script_file], stdout=PIPE, stderr=PIPE, universal_newlines=True)

        out, err = process.communicate()
        # grab the return code. PBS returns 0 if the job was successful
        queue_id = None
        if process.returncode == 0:
            try:
                # output should of the form '2561553.sdb' or '352353.jessup' - just grab the first part for job id
                queue_id = int(out.split('.')[0])
            except:
                # probably error parsing job code
                logger.critical("Could not parse job id following qsub...")
        return SubmitResults(qid=queue_id, out=out, err=err, process=process)
353020
async def write(self):
        
        connection = self.connection
        commands = self.commands

        # We are going to clobber the commands with the write, so go ahead
        # and ensure that nothing is sitting there from a previous run.
        for c in commands:
            c.result = None

        # build up all commands into a single request to increase network perf
        # send all the commands and catch connection and timeout errors.
        try:
            await connection.send_packed_command(connection.pack_commands([c.args for c in commands]))
        except (ConnectionError, TimeoutError) as e:
            for c in commands:
                c.result = e
456652
def _WriteFlowResponsesAndExpectedUpdates(self, responses, cursor=None):
    """"""

    self._WriteResponses(responses, cursor)

    query = """
      UPDATE flow_requests
      SET responses_expected=%(responses_expected)s
      WHERE
        client_id = %(client_id)s AND
        flow_id = %(flow_id)s AND
        request_id = %(request_id)s
    """

    for r in responses:
      # If the response is a FlowStatus, we have to update the FlowRequest with
      # the number of expected messages.
      if isinstance(r, rdf_flow_objects.FlowStatus):
        args = {
            "client_id": db_utils.ClientIDToInt(r.client_id),
            "flow_id": db_utils.FlowIDToInt(r.flow_id),
            "request_id": r.request_id,
            "responses_expected": r.response_id,
        }
        cursor.execute(query, args)
828033
def WriteFlowProcessingRequests(self, requests):
    """"""
    # If we don't have a handler thread running, we might be able to process the
    # requests inline. If we are not, we start the handler thread for real and
    # queue the requests normally.
    if not self.flow_handler_thread and self.flow_handler_target:
      if self._InlineProcessingOK(requests):
        for r in requests:
          self.flow_handler_target(r)
        return
      else:
        self._RegisterFlowProcessingHandler(self.flow_handler_target)
        self.flow_handler_target = None

    now = rdfvalue.RDFDatetime.Now()
    for r in requests:
      cloned_request = r.Copy()
      cloned_request.timestamp = now
      key = (r.client_id, r.flow_id)
      self.flow_processing_requests[key] = cloned_request