Select task to fetch & process
171624
def _check_select(self):
        ''''''
        while self._send_buffer:
            _task = self._send_buffer.pop()
            try:
                # use force=False here to prevent automatic send_buffer append and get exception
                self.send_task(_task, False)
            except Queue.Full:
                self._send_buffer.append(_task)
                break

        if self.out_queue.full():
            return {}

        taskids = []
        cnt = 0
        cnt_dict = dict()
        limit = self.LOOP_LIMIT

        # dynamic assign select limit for each project, use qsize as weight
        project_weights, total_weight = dict(), 0
        for project in itervalues(self.projects):  # type:Project
            if not project.active:
                continue
            # only check project pause when select new tasks, cronjob and new request still working
            if project.paused:
                continue
            if project.waiting_get_info:
                continue

            # task queue
            task_queue = project.task_queue  # type:TaskQueue
            pro_weight = task_queue.size()
            total_weight += pro_weight
            project_weights[project.name] = pro_weight
            pass

        min_project_limit = int(limit / 10.)  # ensure minimum select limit for each project
        max_project_limit = int(limit / 3.0)  # ensure maximum select limit for each project

        for pro_name, pro_weight in iteritems(project_weights):
            if cnt >= limit:
                break

            project = self.projects[pro_name]  # type:Project

            # task queue
            task_queue = project.task_queue
            task_queue.check_update()
            project_cnt = 0

            # calculate select limit for project
            if total_weight < 1 or pro_weight < 1:
                project_limit = min_project_limit
            else:
                project_limit = int((1.0 * pro_weight / total_weight) * limit)
                if project_limit < min_project_limit:
                    project_limit = min_project_limit
                elif project_limit > max_project_limit:
                    project_limit = max_project_limit

            # check send_buffer here. when not empty, out_queue may blocked. Not sending tasks
            while cnt < limit and project_cnt < project_limit:
                taskid = task_queue.get()
                if not taskid:
                    break

                taskids.append((project.name, taskid))
                if taskid != 'on_finished':
                    project_cnt += 1
                cnt += 1

            cnt_dict[project.name] = project_cnt
            if project_cnt:
                project._selected_tasks = True
                project._send_finished_event_wait = 0

            # check and send finished event to project
            if not project_cnt and len(task_queue) == 0 and project._selected_tasks:
                # wait for self.FAIL_PAUSE_NUM steps to make sure all tasks in queue have been processed
                if project._send_finished_event_wait < self.FAIL_PAUSE_NUM:
                    project._send_finished_event_wait += 1
                else:
                    project._selected_tasks = False
                    project._send_finished_event_wait = 0

                    self._postpone_request.append({
                        'project': project.name,
                        'taskid': 'on_finished',
                        'url': 'data:,on_finished',
                        'process': {
                            'callback': 'on_finished',
                        },
                        "schedule": {
                            "age": 0,
                            "priority": 9,
                            "force_update": True,
                        },
                    })

        for project, taskid in taskids:
            self._load_put_task(project, taskid)

        return cnt_dict
1148842
def async_fetch_in_batch(handles, batch_size = 1000, &block)
      raise "No block given for the batch fetch request!" unless block_given?
      # Can't get data from an unfinished query
      unless async_is_complete?(handles)
        raise "Can't perform fetch on a query in state: #{async_state(handles)}"
      end

      # Now let's iterate over the results
      loop do
        rows = fetch_rows(prepare_operation_handle(handles), :next, batch_size)
        break if rows.empty?
        yield rows
      end
    end
921896
def process(self):
        """"""
        if WINDOWS:
            select_inputs = []
            for i in self.inputs:
                if not isinstance(i, SelectableObject):
                    warning("Unknown ignored object type: %s", type(i))
                elif i.__selectable_force_select__:
                    # Then use select.select
                    select_inputs.append(i)
                elif not self.remain and i.check_recv():
                    self.results.append(i)
                else:
                    i.wait_return(self._exit_door)
            if select_inputs:
                # Use default select function
                self.results.extend(select(select_inputs, [], [], self.remain)[0])  # noqa: E501
            if not self.remain:
                return self.results

            threading.Thread(target=self._timeout_thread, args=(self.remain,)).start()  # noqa: E501
            if not self._ended:
                self.available_lock.acquire()
            return self.results
        else:
            r, _, _ = select(self.inputs, [], [], self.remain)
            return r
1193264
def fetch(*queues)
      job = nil
      transaction do
        command("FETCH", *queues)
        job = result!
      end
      JSON.parse(job) if job
    end
1105384
def batch!(request, request_options = {})
      res = batch(request, request_options)
      wait_task(res['taskID'], WAIT_TASK_DEFAULT_TIME_BEFORE_RETRY, request_options)
      res
    end
949041
private void processTasks() throws SQLException
   {
      List<Row> bars = getRows("select * from bar where projid=?", m_projectID);
      List<Row> expandedTasks = getRows("select * from expanded_task where projid=?", m_projectID);
      List<Row> tasks = getRows("select * from task where projid=?", m_projectID);
      List<Row> milestones = getRows("select * from milestone where projid=?", m_projectID);
      m_reader.processTasks(bars, expandedTasks, tasks, milestones);
   }
978824
def fetch(opts):
    
    resources = _load(opts.resources, opts.output_dir)
    if opts.all:
        opts.resource_names = ALL
    reporthook = None if opts.quiet else lambda name: print('Fetching {}...'.format(name))
    if opts.verbose:
        backend.VERBOSE = True
    _fetch(resources, opts.resource_names, opts.mirror_url, opts.force, reporthook)
    return verify(opts)
244393
function pull() {
  var tasks = [],
      cwd = process.cwd();

  findRepos(function(err, repos) {
    if (err) return console.log(err);
    repos.forEach(function(repo) {
      tasks.push(function(done) {
        repo.status(function(err, status) {
          var name = status.repo.path.replace(cwd + '/', '');
          if (!status.clean) {
            console.log('Skipped '.red + name.bold + ' (not clean)'.grey);
            done();
          }
          else {
            console.log('Pulling '.green + name.bold);
            process.chdir(status.repo.path);
            exec('git fetch', function(err, stdout, stderr) {
              if (err) return done(stderr);
              if (stdout && argv.v) console.log(indent(stdout));
              exec('git rebase', function(err, stdout, stderr) {
                if (err) return done(stderr);
                if (stdout && argv.v) console.log(indent(stdout));
                done(null);
              });
            });
          }
        });
      });
    });

    async.series(tasks, function(err, results) {
      if (err) {
        console.log('Error:'.red);
        console.log(err);
      }
      else {
        console.log('Done!');
      }
    });
  });
}
170703
def run(self):
        """"""
        try:
            while not self.stopped:
                scraperobj = jobs.get(False)
                self.setName(scraperobj.getName())
                try:
                    self.getStrips(scraperobj)
                finally:
                    jobs.task_done()
                    self.setName(self.origname)
        except Empty:
            pass
        except KeyboardInterrupt:
            thread.interrupt_main()
1263060
def phantomjs_fetch(self, url, task):
        ''''''
        start_time = time.time()
        self.on_fetch('phantomjs', task)
        handle_error = lambda x: self.handle_error('phantomjs', url, task, start_time, x)

        # check phantomjs proxy is enabled
        if not self.phantomjs_proxy:
            result = {
                "orig_url": url,
                "content": "phantomjs is not enabled.",
                "headers": {},
                "status_code": 501,
                "url": url,
                "time": time.time() - start_time,
                "cookies": {},
                "save": task.get('fetch', {}).get('save')
            }
            logger.warning("[501] %s:%s %s 0s", task.get('project'), task.get('taskid'), url)
            raise gen.Return(result)

        # setup request parameters
        fetch = self.pack_tornado_request_parameters(url, task)
        task_fetch = task.get('fetch', {})
        for each in task_fetch:
            if each not in fetch:
                fetch[each] = task_fetch[each]

        # robots.txt
        if task_fetch.get('robots_txt', False):
            user_agent = fetch['headers']['User-Agent']
            can_fetch = yield self.can_fetch(user_agent, url)
            if not can_fetch:
                error = tornado.httpclient.HTTPError(403, 'Disallowed by robots.txt')
                raise gen.Return(handle_error(error))

        request_conf = {
            'follow_redirects': False
        }
        request_conf['connect_timeout'] = fetch.get('connect_timeout', 20)
        request_conf['request_timeout'] = fetch.get('request_timeout', 120) + 1

        session = cookies.RequestsCookieJar()
        if 'Cookie' in fetch['headers']:
            c = http_cookies.SimpleCookie()
            try:
                c.load(fetch['headers']['Cookie'])
            except AttributeError:
                c.load(utils.utf8(fetch['headers']['Cookie']))
            for key in c:
                session.set(key, c[key])
            del fetch['headers']['Cookie']
        if 'cookies' in fetch:
            session.update(fetch['cookies'])
            del fetch['cookies']

        request = tornado.httpclient.HTTPRequest(url=fetch['url'])
        cookie_header = cookies.get_cookie_header(session, request)
        if cookie_header:
            fetch['headers']['Cookie'] = cookie_header

        # making requests
        fetch['headers'] = dict(fetch['headers'])
        try:
            request = tornado.httpclient.HTTPRequest(
                url=self.phantomjs_proxy, method="POST",
                body=json.dumps(fetch), **request_conf)
        except Exception as e:
            raise gen.Return(handle_error(e))

        try:
            response = yield gen.maybe_future(self.http_client.fetch(request))
        except tornado.httpclient.HTTPError as e:
            if e.response:
                response = e.response
            else:
                raise gen.Return(handle_error(e))

        if not response.body:
            raise gen.Return(handle_error(Exception('no response from phantomjs: %r' % response)))

        result = {}
        try:
            result = json.loads(utils.text(response.body))
            assert 'status_code' in result, result
        except Exception as e:
            if response.error:
                result['error'] = utils.text(response.error)
            raise gen.Return(handle_error(e))

        if result.get('status_code', 200):
            logger.info("[%d] %s:%s %s %.2fs", result['status_code'],
                        task.get('project'), task.get('taskid'), url, result['time'])
        else:
            logger.error("[%d] %s:%s %s, %r %.2fs", result['status_code'],
                         task.get('project'), task.get('taskid'),
                         url, result['content'], result['time'])

        raise gen.Return(result)
774676
function task(callback) {
	// spawn phantomas process
	var child = phantomas(url, options, function(err, data, results) {
		callback(
			null, // pass null even in case of an error to continue async.series processing (issue #380)
			[err, results]
		);
	});

	child.on('progress', function(progress, inc) {
		if (bar) {
			bar.tick(inc);
		}
	});

	// pipe --verbose messages to stderr
	child.stderr.pipe(process.stderr);
}
822351
def on_task_status(self, task):
        ''''''
        try:
            procesok = task['track']['process']['ok']
            if not self.projects[task['project']].task_queue.done(task['taskid']):
                logging.error('not processing pack: %(project)s:%(taskid)s %(url)s', task)
                return None
        except KeyError as e:
            logger.error("Bad status pack: %s", e)
            return None

        if procesok:
            ret = self.on_task_done(task)
        else:
            ret = self.on_task_failed(task)

        if task['track']['fetch'].get('time'):
            self._cnt['5m_time'].event((task['project'], 'fetch_time'),
                                       task['track']['fetch']['time'])
        if task['track']['process'].get('time'):
            self._cnt['5m_time'].event((task['project'], 'process_time'),
                                       task['track']['process'].get('time'))
        self.projects[task['project']].active_tasks.appendleft((time.time(), task))
        return ret
1181616
def get_tasks():
    """"""
    from paver.tasks import environment
    for tsk in environment.get_tasks():
        print(tsk.shortname)
407490
def stream_task_output(session, task_id, file_name,
                       blocksize=DEFAULT_DOWNLOAD_BLOCK_SIZE):
    
    logger.debug('Streaming {} from task {}'.format(file_name, task_id))
    offset = 0
    contents = '[PLACEHOLDER]'
    while contents:
        contents = session.downloadTaskOutput(task_id, file_name, offset,
                                              blocksize)
        offset += len(contents)
        if contents:
            yield contents

    logger.debug('Finished streaming {} from task {}'.format(file_name, task_id))
81034
def task(self):
        
        # If we have no .task_id, this is a no-op to return None.
        if not self.task_id:
            return defer.succeed(None)
        return self.connection.getTaskInfo(self.task_id)
1019422
public static void processTask(Task task) {
    logger.finest("Processing task " + task);
    try {
      switch (task.getType()) {
        case RUN_JOB:
          runJob((RunJobTask) task);
          break;
        case HANDLE_SLOT_FILLED:
          handleSlotFilled((HandleSlotFilledTask) task);
          break;
        case FINALIZE_JOB:
          finalizeJob((FinalizeJobTask) task);
          break;
        case FAN_OUT:
          handleFanoutTaskOrAbandonTask((FanoutTask) task);
          break;
        case CANCEL_JOB:
          cancelJob((CancelJobTask) task);
          break;
        case DELETE_PIPELINE:
          DeletePipelineTask deletePipelineTask = (DeletePipelineTask) task;
          try {
            backEnd.deletePipeline(
                deletePipelineTask.getRootJobKey(), deletePipelineTask.shouldForce(), false);
          } catch (Exception e) {
            logger.log(Level.WARNING, "DeletePipeline operation failed.", e);
          }
          break;
        case HANDLE_CHILD_EXCEPTION:
          handleChildException((HandleChildExceptionTask) task);
          break;
        case DELAYED_SLOT_FILL:
          handleDelayedSlotFill((DelayedSlotFillTask) task);
          break;
        default:
          throw new IllegalArgumentException("Unrecognized task type: " + task.getType());
      }
    } catch (AbandonTaskException ate) {
      // return 200;
    }
  }
319550
def _fetch_page_async(self, page_size, **q_options):
    """"""
    q_options.setdefault('batch_size', page_size)
    q_options.setdefault('produce_cursors', True)
    it = self.iter(limit=page_size + 1, **q_options)
    results = []
    while (yield it.has_next_async()):
      results.append(it.next())
      if len(results) >= page_size:
        break
    try:
      cursor = it.cursor_after()
    except datastore_errors.BadArgumentError:
      cursor = None
    raise tasklets.Return(results, cursor, it.probably_has_next())