Get all strips from a scraper.
2170
def _getStrips(self, scraperobj):
        """"""
        if self.options.all or self.options.cont:
            numstrips = None
        elif self.options.numstrips:
            numstrips = self.options.numstrips
        else:
            # get current strip
            numstrips = 1
        try:
            if scraperobj.isComplete(self.options.basepath):
                out.info(u"All comics are already downloaded.")
                return 0
            for strip in scraperobj.getStrips(numstrips):
                skipped = self.saveComicStrip(strip)
                if skipped and self.options.cont:
                    # stop when retrieval skipped an image for one comic strip
                    out.info(u"Stop retrieval because image file already exists")
                    break
                if self.stopped:
                    break
            if self.options.all and not (self.errors or self.options.dry_run or
                                    self.options.cont or scraperobj.indexes):
                scraperobj.setComplete(self.options.basepath)
        except Exception as msg:
            out.exception(msg)
            self.errors += 1
292708
function getScrapersResults (SCRAPERS, urls, htmls) {
  return Promise.all(
    SCRAPERS.map(SCRAPER => {
      return getScraperResults(SCRAPER, urls, htmls).then(results => {
        return {
          name: SCRAPER.name,
          results: results
        }
      })
    })
  )
}
694254
def _stripStrList(self, raw_str, stop_strs):
        '''
        
        '''
        if type(stop_strs) == list:
            for word in stop_strs:
                raw_str = self._stripStr(raw_str, word)
            return raw_str
        else:
            raise Exception('stop_words must be list!')
484254
def strip_exts(s, exts):
    
    f_split = os.path.splitext(s)
    if f_split[1] in exts:
        return f_split[0]
    else:
        return s
1303483
function scrapeMetadata (html, rules) {
  rules = rules || RULES
  const keys = Object.keys(rules)
  const $ = cheerio.load(html)
  const promises = keys.map(function (key) {
    return scrapeMetadatum($, '', rules[key])
  })

  return Promise.all(promises).then(function (values) {
    return keys.reduce(function (memo, key, i) {
      memo[key] = values[i]
      return memo
    }, {})
  })
}
136214
function scraper(options){

  if(typeof options === 'undefined'){
    throw new Error('settings object is required.');
  }

  if(typeof options.url === 'undefined'){
    throw new Error('settings.url is required.');
  }

  if(typeof options.paths === 'undefined'){
    throw new Error('settings.paths is required');
  }

  // options
  options = assign({
    url: null,
    paths: [],
    keyword: '@component',
    block: '{{block}}',
    output: null,
    complete: function() {}
  }, options);

  var counter = 0;

  // register files
  var register = {};

  
  function getHTML(i) {

    // complete
    if (i === options.paths.length) {
      init();
      return;
    }

    request.get(options.url + options.paths[i], (error, response, data) => {
      if (error) {
        console.log(error);
        getHTML(++i);
        return;
      }

      if (parser.scan(data, options.keyword)) {
        register[options.paths[i]] = data;
      }
      getHTML(++i);
    });
  }

  getHTML(0);


  // create readable stream
  var stream = through2({ objectMode: true },
    function(chunk, enc, next){
      this.push(chunk);
      next();
    },
    function(cb){
      cb();
    }
  );


  /**
   * Init
   */

  function init(){
    var output;
    var components = [];

    // write stream to output file
    if(options.output){

      // create directory if it doesn't exist
      mkdirp.sync(path.dirname(options.output));

      // create writable stream
      output = fs.createWriteStream(options.output);
      output.write('[');

      output.on('close', function(){
        stream.emit('complete');
      });

      output.on('finish', function() {
        if (options.complete && typeof options.complete === 'function') {
          options.complete(components);
        }
      });
    }

    // get all files
    var file;

    // collect docs for all files
    for(file in register){
      var docs = parser.getComponents(register[file], file, options);

      docs.forEach(function(docItem) {
        // omit first comma
        if(counter !== 0 && options.output){
          output.write(',');
        }
        // add object to stream
        stream.push(docItem);
        if(options.output){
          // send to output
          output.write(JSON.stringify(docItem));
        }
        components.push(docItem);
        // up counter
        ++counter;
      });
    }

    // end json array stream
    if(options.output){
      output.write(']');
      output.end();
    } else {
      if (options.complete && typeof options.complete === 'function') {        
        options.complete(components);
      }
    }

    // end stream
    stream.push(null);
  }

  return stream;

}
1367540
async def fetchall(self):
        """"""
        try:
            rows = await self._cursor.fetchall()
        except AttributeError:
            self._non_result()
        else:
            ret = self._process_rows(rows)
            await self.close()
            return ret
710504
def find_all(self, cls):
        """"""
        final_results = []
        for db_result in read_by_indexes(cls.get_table_name(), []):
            obj = cls.from_data(db_result['value'])
            final_results.append(obj)

        return final_results
870610
def all(cls, api_key=None, idempotency_key=None,
            stripe_account=None, **params):
        """"""
        url = cls.class_url()
        return make_request(
            cls, 'get', url, stripe_acconut=None, params=params)
328666
def _strip(string, pattern):
    """"""
    m = re.compile(pattern).search(string)

    if m:
        return string[0:m.start()] + string[m.end():len(string)]
    else:
        return string
764660
def get_new_requests(self):
        

        content_type = self.__queue_item.response.headers.get('content-type')
        scrapers = self.__get_all_scrapers()
        new_requests = []

        for scraper in scrapers:
            instance = scraper(self.__options, self.__queue_item)
            if self.__content_type_matches(content_type, instance.content_types):
                new_requests.extend(instance.get_requests())

        return new_requests
27477
def pull_all(collector, image, **kwargs):
    """"""
    images = collector.configuration["images"]

    for layer in Builder().layered(images, only_pushable=True):
        for image_name, image in layer:
            log.info("Pulling %s", image_name)
            pull(collector, image, **kwargs)
579837
def fetch_all(self, R, depth=1, **kwargs):
        ""
        d, e = self._fetcher.fetch_all(R, depth, kwargs)
        if e: raise e
        return d
220368
def strip_tags(cls, html):
        
        s = cls()
        s.feed(html)

        return s.get_data()
118085
def get_rows(self):
        """"""
        table = self.soup.find_all('tr')[1:-3]
        return [row for row in table if row.contents[3].string]
109916
def _get_all_slots(self):
        """"""
        all_slots = (getattr(cls, '__slots__', [])
                         for cls in self.__class__.__mro__)
        return set(slot for slots in all_slots for slot in slots)
180106
def get_scraperclasses():
    
    global _scraperclasses
    if _scraperclasses is None:
        out.debug(u"Loading comic modules...")
        modules = loader.get_modules('plugins')
        plugins = loader.get_plugins(modules, Scraper)
        _scraperclasses = list(plugins)
        check_scrapers()
        out.debug(u"... %d modules loaded." % len(_scraperclasses))
    return _scraperclasses