Process robots.txt.

        Coroutine.
501245
def _process_robots(self):
        '''
        '''
        try:
            self._item_session.request = request = self._new_initial_request(with_body=False)
            verdict, reason = (yield from self._should_fetch_reason_with_robots(
                request))
        except REMOTE_ERRORS as error:
            _logger.error(
                _('Fetching robots.txt for ‘{url}’ '
                  'encountered an error: {error}'),
                url=self._next_url_info.url, error=error
            )
            self._result_rule.handle_error(self._item_session, error)

            wait_time = self._result_rule.get_wait_time(
                self._item_session, error=error
            )

            if wait_time:
                _logger.debug('Sleeping {0}.', wait_time)
                yield from asyncio.sleep(wait_time)

            return False
        else:
            _logger.debug('Robots filter verdict {} reason {}', verdict, reason)

            if not verdict:
                self._item_session.skip()
                return False

        return True
1195896
async def execute_process(*cmd, log=None, loop=None):
    '''
    

    '''
    p = await asyncio.create_subprocess_exec(
        *cmd,
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
        loop=loop)
    stdout, stderr = await p.communicate()
    if log:
        log.debug("Exec %s -> %d", cmd, p.returncode)
        if stdout:
            log.debug(stdout.decode('utf-8'))
        if stderr:
            log.debug(stderr.decode('utf-8'))
    return p.returncode == 0
726564
async def _thread_coro(self, *args):
        
        return await self._loop.run_in_executor(
            self._executor, self._function, *args)
842932
def _bg(self, coro: coroutine) -> asyncio.Task:
        """"""
        async def runner():
            try:
                await coro
            except:
                self._log.exception("async: Coroutine raised exception")
        return asyncio.ensure_future(runner())
236541
def submit(self, fn, *args, **kwargs):
        """"""
        corofn = asyncio.coroutine(lambda: fn(*args, **kwargs))
        return run_coroutine_threadsafe(corofn(), self.loop)
570518
async def iter_lines(
            self,
            *cmds: str,
            stream: str='both') -> AsyncGenerator[str, None]:
        """"""
        sps = self.spawn(*cmds)
        if stream == 'both':
            agen = amerge(
                amerge(*[sp.stdout for sp in sps]),
                amerge(*[sp.stderr for sp in sps]))
        elif stream == 'stdout':
            agen = amerge(*[sp.stdout for sp in sps])
        elif stream == 'stderr':
            agen = amerge(*[sp.stderr for sp in sps])
        else:
            raise SublemonRuntimeError(
                'Invalid `stream` kwarg received: `' + str(stream) + '`')
        async for line in agen:
            yield line.decode('utf-8').rstrip()
865034
def loop(bot, config, interval, settings):
    """'"""
    print_options(bot, config, settings)
    click.echo(f'- Interval: {interval}s')
    click.echo()
    bot_task = BotTask(bot, config)
    bot_task.run_loop(interval)
1153707
def pipe(self, transformer):
		""""""
		if self.next:
			return

		stream = Stream()
		self.next = stream
		stream.prev = self

		self.transformer = transformer
		transformer.stream = self
		transformer.piped()

		for file in self.files:
			future = asyncio.ensure_future(self.transformer.transform(file))
			future.add_done_callback(self.handle_transform)	

		self.onpiped.set_result(None)
		self.flush_if_ended()

		return stream
18821
def Routine(coroutine, scheduler, asyncStart = True, container = None, manualStart = False, daemon = False):
    
    def run():
        iterator = _await(coroutine)
        iterself = yield
        if manualStart:
            yield
        try:
            if asyncStart:
                scheduler.yield_(iterself)
                yield
            if container is not None:
                container.currentroutine = iterself
            if daemon:
                scheduler.setDaemon(iterself, True)
            try:
                matchers = next(iterator)
            except StopIteration:
                return
            while matchers is None:
                scheduler.yield_(iterself)
                yield
                try:
                    matchers = next(iterator)
                except StopIteration:
                    return
            try:
                scheduler.register(matchers, iterself)
            except Exception:
                try:
                    iterator.throw(IllegalMatchersException(matchers))
                except StopIteration:
                    pass
                raise
            while True:
                try:
                    etup = yield
                except GeneratorExit_:
                    raise
                except:
                    #scheduler.unregister(matchers, iterself)
                    lmatchers = matchers
                    t,v,tr = sys.exc_info()  # @UnusedVariable
                    if container is not None:
                        container.currentroutine = iterself
                    try:
                        matchers = iterator.throw(t,v)
                    except StopIteration:
                        return
                else:
                    #scheduler.unregister(matchers, iterself)
                    lmatchers = matchers
                    if container is not None:
                        container.currentroutine = iterself
                    try:
                        matchers = iterator.send(etup)
                    except StopIteration:
                        return
                while matchers is None:
                    scheduler.yield_(iterself)
                    yield
                    try:
                        matchers = next(iterator)
                    except StopIteration:
                        return
                try:
                    if hasattr(matchers, 'two_way_difference'):
                        reg, unreg = matchers.two_way_difference(lmatchers)
                    else:
                        reg = set(matchers).difference(lmatchers)
                        unreg = set(lmatchers).difference(matchers)
                    scheduler.register(reg, iterself)
                    scheduler.unregister(unreg, iterself)
                except Exception:
                    try:
                        iterator.throw(IllegalMatchersException(matchers))
                    except StopIteration:
                        pass
                    raise
        finally:
            # iterator.close() can be called in other routines, we should restore the currentroutine variable
            if container is not None:
                lastcurrentroutine = getattr(container, 'currentroutine', None)
                container.currentroutine = iterself
            else:
                lastcurrentroutine = None
            _close_generator(coroutine)
            if container is not None:
                container.currentroutine = lastcurrentroutine
            scheduler.unregisterall(iterself)
    r = generatorwrapper(run())
    next(r)
    r.send(r)
    return r
76103
def txt_line_iterator(path):
  """"""
  with tf.gfile.Open(path) as f:
    for line in f:
      yield line.strip()
1148346
def process(self, sched, coro):
        """"""

        if self.prio == priority.DEFAULT:
            self.prio = sched.default_priority
715639
function publishRobotsTxt () {
    var path = 'robots.txt';
    return fs
        .statAsync(path)
        .then(function (pStat) {
            if (!pStat.isFile()) {
                // go to our default course of action, that is create robots.txt file
                throw new Error ();
            }
        })
        .catch(Error, function () {
            return fs
                .openAsync(path, 'w')
                .then(function () {
                    return fs.writeFileAsync(path, 'User-agent: *\nSitemap: ' + config.siteUrl + '/sitemap.xml');
                })
                .catch(function (pErr) {
                    log.error(pErr);
                });
        });
}
16806
def _parse_coroutine(self):
        
        while True:
            d = yield

            if d == int2byte(0):
                pass  # NOP

            # Go to state escaped.
            elif d == IAC:
                d2 = yield

                if d2 == IAC:
                    self.received_data(d2)

                # Handle simple commands.
                elif d2 in (NOP, DM, BRK, IP, AO, AYT, EC, EL, GA):
                    self.command_received(d2, None)

                # Handle IAC-[DO/DONT/WILL/WONT] commands.
                elif d2 in (DO, DONT, WILL, WONT):
                    d3 = yield
                    self.command_received(d2, d3)

                # Subnegotiation
                elif d2 == SB:
                    # Consume everything until next IAC-SE
                    data = []

                    while True:
                        d3 = yield

                        if d3 == IAC:
                            d4 = yield
                            if d4 == SE:
                                break
                            else:
                                data.append(d4)
                        else:
                            data.append(d3)

                    self.negotiate(b''.join(data))
            else:
                self.received_data(d)
522332
async def one(s: Sublemon):
    """"""
    shell_cmds = [
        'sleep 1 && echo subprocess 1 in coroutine one',
        'sleep 1 && echo subprocess 2 in coroutine one']
    async for line in s.iter_lines(*shell_cmds):
        print(line)
1343931
def perform_action(
            self, action, machines, params, progress_title, success_title):
        """"""
        if len(machines) == 0:
            return 0
        with utils.Spinner() as context:
            return self._async_perform_action(
                context, action, list(machines), params,
                progress_title, success_title)
514498
def run(crawler):
    """"""
    crawler = get_crawler(crawler)
    crawler.run()
    if is_sync_mode():
        TaskRunner.run_sync()
241288
private Process executeToAstProcess(String argument) throws IOException {
        try {
            Process p = new ProcessBuilder("python", codeGenBinDirectory + "/jp-to-ast.py", argument).start();
            p.waitFor();
            return p;
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new RuntimeException(e);
        }
    }