Callback for self._client.new_job
763742
def _callback(self, bjobid, result, grade, problems, tests, custom, archive, stdout, stderr):
        
        self._jobs_done[str(bjobid)] = (result, grade, problems, tests, custom, archive, stdout, stderr)
        self._waiting_jobs.remove(str(bjobid))
233204
def run_job(self, project_id, body, dataset=None):
        

        if dataset and not self.dataset_exists(dataset):
            self.make_dataset(dataset)

        new_job = self.client.jobs().insert(projectId=project_id, body=body).execute()
        job_id = new_job['jobReference']['jobId']
        logger.info('Started import job %s:%s', project_id, job_id)
        while True:
            status = self.client.jobs().get(projectId=project_id, jobId=job_id).execute(num_retries=10)
            if status['status']['state'] == 'DONE':
                if status['status'].get('errorResult'):
                    raise Exception('BigQuery job failed: {}'.format(status['status']['errorResult']))
                return

            logger.info('Waiting for job %s:%s to complete...', project_id, job_id)
            time.sleep(5)
276448
def add_job(self, data, persist=True):
        '''
        
        '''

        # we don't do any checking here besides making sure its a dict.
        # eval() already does for us and raises errors accordingly
        if not isinstance(data, dict):
            raise ValueError('Scheduled jobs have to be of type dict.')
        if not len(data) == 1:
            raise ValueError('You can only schedule one new job at a time.')

        # if enabled is not included in the job,
        # assume job is enabled.
        for job in data:
            if 'enabled' not in data[job]:
                data[job]['enabled'] = True

        new_job = next(six.iterkeys(data))

        if new_job in self._get_schedule(include_opts=False):
            log.warning("Cannot update job %s, it's in the pillar!", new_job)

        elif new_job in self.opts['schedule']:
            log.info('Updating job settings for scheduled job: %s', new_job)
            self.opts['schedule'].update(data)

        else:
            log.info('Added new job %s to scheduler', new_job)
            self.opts['schedule'].update(data)

        # Fire the complete event back along with updated list of schedule
        evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False)
        evt.fire_event({'complete': True,
                        'schedule': self._get_schedule()},
                       tag='/salt/minion/minion_schedule_add_complete')

        if persist:
            self.persist()
269642
def notify(self, new_jobs_count):
        

        if not self.use_notify():
            return

        # Not really useful to send more than 100 notifs (to be configured)
        count = min(new_jobs_count, 100)

        notify_key = redis_key("notify", self)

        context.connections.redis.lpush(notify_key, *([1] * count))
        context.connections.redis.expire(notify_key, max(1, int(context.get_current_config()["max_latency"] * 2)))
1237844
def job_success(self, job, queue, job_result):
        

        # display what was done
        obj = job.get_object()
        message = '[%s|%s] %s [%s]' % (queue.name.hget(),
                                       obj.pk.get(),
                                       job_result,
                                       threading.current_thread().name)
        self.log(message)

        # default stuff: update job and queue statuses, and do logging
        super(FullNameWorker, self).job_success(job, queue, job_result)

        # update the queue's dates
        queue_fields_to_update = {
            'last_job_date': str(datetime.utcnow())
        }
        if not queue.first_job_date.hget():
            queue_fields_to_update['first_job_date'] = queue_fields_to_update['last_job_date']
        queue.hmset(**queue_fields_to_update)

        # update the jobs counter on the queue
        queue.jobs_counter.hincrby(1)

        # save a ref to the job to display at the end of this example script
        self.jobs.append(int(job.pk.get()))
243374
def _start_new_worker_process(self, server_socket):
        """"""
        from multiprocessing import Process
        p = Process(target=forked_child, args=self._get_child_args(server_socket))
        p.start()
        return p
875707
def brozzler_new_job(argv=None):
    '''
    
    '''
    argv = argv or sys.argv
    arg_parser = argparse.ArgumentParser(
            prog=os.path.basename(argv[0]),
            description='brozzler-new-job - queue new job with brozzler',
            formatter_class=BetterArgumentDefaultsHelpFormatter)
    arg_parser.add_argument(
            'job_conf_file', metavar='JOB_CONF_FILE',
            help='brozzler job configuration file in yaml')
    add_rethinkdb_options(arg_parser)
    add_common_options(arg_parser, argv)

    args = arg_parser.parse_args(args=argv[1:])
    configure_logging(args)

    rr = rethinker(args)
    frontier = brozzler.RethinkDbFrontier(rr)
    try:
        brozzler.new_job_file(frontier, args.job_conf_file)
    except brozzler.InvalidJobConf as e:
        print('brozzler-new-job: invalid job file:', args.job_conf_file, file=sys.stderr)
        print('  ' + yaml.dump(e.errors).rstrip().replace('\n', '\n  '), file=sys.stderr)
        sys.exit(1)
427295
def insert(cls, jobs_data, queue=None, statuses_no_storage=None, return_jobs=True, w=None, j=None):
        

        now = datetime.datetime.utcnow()
        for data in jobs_data:
            if data["status"] == "started":
                data["datestarted"] = now

        no_storage = (statuses_no_storage is not None) and ("started" in statuses_no_storage)
        if no_storage and return_jobs:
            for data in jobs_data:
                data["_id"] = ObjectId()  # Give the job a temporary ID
        else:
            inserted = context.connections.mongodb_jobs.mrq_jobs.insert(
                jobs_data,
                manipulate=True,
                w=w,
                j=j
            )

        if return_jobs:
            jobs = []
            for data in jobs_data:
                job = cls(data["_id"], queue=queue)
                job.set_data(data)
                job.statuses_no_storage = statuses_no_storage
                job.stored = (not no_storage)
                if data["status"] == "started":
                    job.datestarted = data["datestarted"]
                jobs.append(job)

            return jobs
        else:
            return inserted
705820
def job(func_or_queue=None, connection=None, *args, **kwargs):
    
    if callable(func_or_queue):
        func = func_or_queue
        queue = 'default'
    else:
        func = None
        queue = func_or_queue or 'default'

    if not isinstance(queue, basestring):
        queue = unicode(queue)

    try:
        queue = get_queue(queue)
        if connection is None:
            connection = queue.connection
    except KeyError:
        pass

    decorator = _job(queue, connection=connection, *args, **kwargs)
    if func:
        return decorator(func)
    return decorator
990658
function (jobName, customParams, callback) {
      [jobName, customParams, callback] = doArgs(arguments, ['string', ['object', {}], 'function']);

      doRequest({
        urlPattern: [LAST_BUILD, jobName]
      }, customParams, callback);
    }
417882
def create_new_update_job_from_an_existing_job(user, job_id):
    """"""

    values = {
        'id': utils.gen_uuid(),
        'created_at': datetime.datetime.utcnow().isoformat(),
        'updated_at': datetime.datetime.utcnow().isoformat(),
        'etag': utils.gen_etag(),
        'status': 'new'
    }

    original_job_id = job_id
    original_job = v1_utils.verify_existence_and_get(original_job_id,
                                                     models.JOBS)

    if not user.is_in_team(original_job['team_id']):
        raise dci_exc.Unauthorized()

    # get the remoteci
    remoteci_id = str(original_job['remoteci_id'])
    remoteci = v1_utils.verify_existence_and_get(remoteci_id,
                                                 models.REMOTECIS)
    values.update({'remoteci_id': remoteci_id})

    # get the associated topic
    topic_id = str(original_job['topic_id'])
    v1_utils.verify_existence_and_get(topic_id, models.TOPICS)

    values.update({
        'user_agent': flask.request.environ.get('HTTP_USER_AGENT'),
        'client_version': flask.request.environ.get(
            'HTTP_CLIENT_VERSION'
        ),
    })

    values = _build_job(topic_id, remoteci, [], values,
                        update_previous_job_id=original_job_id)

    return flask.Response(json.dumps({'job': values}), 201,
                          headers={'ETag': values['etag']},
                          content_type='application/json')
1371913
function (jobName, customParams, callback) {
      [jobName, customParams, callback] = doArgs(arguments, ['string', ['object', {}], 'function']);

      doRequest({
        urlPattern: [LAST_COMPLETED_BUILD, jobName]
      }, customParams, callback);
    }
402172
function JobQueue (job, callback) {
     // ignore null job but notify the queue.
     var device;

    if (job === null) {
         callback (null);
         return;
    }
    
    var gateway = job.gateway;
    var status = 0;
    
    // set few defaults
    if (job.timeout === undefined) job.timeout=0;  // no retry
    if (job.retry   === undefined) job.retry  =0;  // retry counter
    if (job.args    === undefined) job.args   =""; // optional arguments
    
    gateway.Debug (4,"Queue Request=%s DevId=%s Command=%s Retry=%d", job.request, job.devId, job.command, job.retry);
    
    // broadcast command loop on active device list to split commands
    if (parseInt (job.devId) === 0) {
        var subrqst=1;
        for (var devid in gateway.activeClients) {
            // we only broadcast to device with a valid TCP session
            if (device && device.socket !== null) {
                var request = {
                  gateway : job.gateway,
                  devId  : gateway.activeClients[devid].devid,
                  command: job.command,
                  request: [job.request, subrqst++],
                  retry  : job.retry, 
                  timeout: job.timeout
                };
            gateway.queue.push (request, JobCallback); // broadcast to all active devices
            }
        }
        var broadcast = {
            gateway  : job.gateway,
            command : 'Broadcast',
            request : [request, subrqst++]
        };
        callback (broadcast);
        return;
    };
    
    // search for devId withing global active devices list
    if (gateway.activeClients [job.devId] !== undefined) {
        // device is present check if it is logged
        gateway.Debug (5,"Queue Request=%s DevId=%s Command=%s args=%s Sent", job.request, job.devId, job.command, job.args);
        device  = gateway.activeClients [job.devId];
        status  = device.RequestAction (job.command, job.args);
    } else {
        gateway.event.emit ("queue","NOTLOG",job);
        status = 1; // device not present in active list
    }

    // process status result and notify callback function
    switch (status) {
        case 0 : // command was accepted by device
            // wait JOB_QUEUE_PAUSE time before processing Job next request
            gateway.queue.pause ();
            setTimeout(function () {gateway.queue.resume();}, JOB_QUEUE_TIMER);
            gateway.event.emit ("queue", "ACCEPT", job);
            break;
            
        case 1: // device not present command will be retry later  
            if (job.timeout === 0) break;     // no retry
            if (job.timeout >= new Date()) {  // check time out and either push or notify users
                job.retry ++;
                gateway.event.emit ("queue", "RETRY", job);
                setTimeout(function () {gateway.queue.push (job, JobCallback);}, JOB_RETRY_TIMER);
            } else {
                gateway.event.emit ("queue", "TIMEOUT", job);
            }                
            break;
               
        case -1 : // device exits but command was refused
            gateway.event.emit ("queue", "REFUSED", job);
            break;
            
        case -2 : // device does not exist in active device list
            gateway.event.emit ("queue", "UNKNOWN", job);
            break;
               
        default:
            gateway.Debug (2,"Hoops invalid status code: Device DevId=%s Status=%s", job.devId, status);
            break;
    }
    callback (job);
}
656402
function jobCallback(job, worker, index) {

        if (index < ALEXA_SIZE) {
            var line = lines[index].trim();

            var split = line.split(',');
            var id = parseInt(split[0]);
            var url = split[1];
            job({
                id : id,
                url : url
            }, function(err) {
                // Lets log if it worked
                total++;
                if (err) {
                    console.log('Problem  #' + worker.id + ': ' + err.message + ' for line: ' + line);
                } else {
                    console.log('    #' + worker.id + '  DONE: ' + line);
                    successful++;
                }
                if (total % 10 === 0) {
                    console.log('################## ' + successful + '/' + total + ' (success/crawled) ################################');
                }
            });
        } else {
            // no more content!
            job(null);
        }
    }
347313
def pass_job(db: JobDB, result_queue: Queue, always_cache=False):
    
    @pull
    def pass_job_stream(job_source):
        """Pull stream instance created by `pass_job`."""
        result_sink = result_queue.sink()

        for message in job_source():
            if message is EndOfQueue:
                return

            key, job = message
            if always_cache or ('store' in job.hints):
                status, retrieved_result = db.add_job_to_db(key, job)

                if status == 'retrieved':
                    result_sink.send(retrieved_result)
                    continue

                elif status == 'attached':
                    continue

            yield message

    return pass_job_stream
1184121
def job_listener(event):
    ''''''
    job_id = event.job.args[0]

    if event.code == events.EVENT_JOB_MISSED:
        db.mark_job_as_missed(job_id)
    elif event.exception:
        if isinstance(event.exception, util.JobError):
            error_object = event.exception.as_dict()
        else:
            error_object = "\n".join(traceback.format_tb(event.traceback) +
                    [repr(event.exception)])
        db.mark_job_as_errored(job_id, error_object)
    else:
        db.mark_job_as_completed(job_id, event.retval)

    api_key = db.get_job(job_id)["api_key"]
    result_ok = send_result(job_id, api_key)

    if not result_ok:
        db.mark_job_as_failed_to_post_result(job_id)

    # Optionally notify tests that job_listener() has finished.
    if "_TEST_CALLBACK_URL" in app.config:
        requests.get(app.config["_TEST_CALLBACK_URL"])
897129
def _handle_job_set(function):
    
    def call(self, job_set=taskhandle.NullJobSet()):
        job_set.started_job(str(self))
        function(self)
        job_set.finished_job()
    return call