Add `urls` to the list that will be prescanned for searches
164520
def add_find_links(self, urls):
        """"""
        for url in urls:
            if (
                self.to_scan is None  # if we have already "gone online"
                or not URL_SCHEME(url)  # or it's a local file/directory
                or url.startswith('file:')
                or list(distros_for_url(url))  # or a direct package link
            ):
                # then go ahead and process it now
                self.scan_url(url)
            else:
                # otherwise, defer retrieval till later
                self.to_scan.append(url)
622524
def prepend_urls(self):
        
        return [
            url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/generate%s$" %
                (self._meta.resource_name, trailing_slash()),
                self.wrap_view('generate'), name="api_tileset_generate"),
            url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/download%s$" %
                (self._meta.resource_name, trailing_slash()),
                self.wrap_view('download'), name="api_tileset_download"),
            url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/status%s$" %
                (self._meta.resource_name, trailing_slash()),
                self.wrap_view('status'), name="api_tileset_status"),
            url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/stop%s$" %
                (self._meta.resource_name, trailing_slash()),
                self.wrap_view('stop'), name="api_tileset_stop"),
        ]
268299
def fetchUrls(cls, url, data, urlSearch):
        """"""
        searchUrls = []
        searches = makeSequence(urlSearch)
        for search in searches:
            for match in search.finditer(data[0]):
                searchUrl = match.group(1)
                if not searchUrl:
                    raise ValueError("Pattern %s matched empty URL at %s." % (search.pattern, url))
                out.debug(u'matched URL %r with pattern %s' % (searchUrl, search.pattern))
                searchUrls.append(normaliseURL(urljoin(data[1], searchUrl)))
            if searchUrls:
                # do not search other links if one pattern matched
                break
        if not searchUrls:
            patterns = [x.pattern for x in searches]
            raise ValueError("Patterns %s not found at URL %s." % (patterns, url))
        return searchUrls
415647
def prefetch_urls(self, urls):
        
        req = {}
        req.update({"urls": urls})

        body = json.dumps(req)
        url = '{0}/v2/tune/prefetch'.format(self.server)
        return self.__post(url, body)
291864
def get_list(self, datatype, url, **kwargs):
        """"""
        search_url = [url, '?']
        kwargs.update({'key': self.api_key})
        search_url.append(urlencode(kwargs))
        data = json.loads(urlopen(''.join(search_url)).read())
        return data[datatype]
975891
def prepare_query_params(**kwargs):
    
    return [
        (sub_key, sub_value)
        for key, value in kwargs.items()
        for sub_key, sub_value in expand(value, key)
        if sub_value is not None
    ]
1041833
def search(cls, query, page=1):
        
        path = '/files/search/{query}/page/{page}'.format(query=query, page=page)
        result = cls.client.request(path)
        files = result['files']
        return [cls(f) for f in files]
1106731
def _add_listing_links(self, response: ListingResponse):
        ''''''
        base_url = response.request.url_info.url

        if self._glob_pattern:
            level = self._item_session.url_record.level
        else:
            level = None

        for file_entry in response.files:
            if self._glob_pattern and \
                    not fnmatch.fnmatchcase(file_entry.name, self._glob_pattern):
                continue

            if file_entry.type == 'dir':
                linked_url = urljoin_safe(base_url, file_entry.name + '/')
            elif file_entry.type in ('file', 'symlink', None):
                if not self._processor.fetch_params.retr_symlinks and \
                        file_entry.type == 'symlink':
                    self._make_symlink(file_entry.name, file_entry.dest)
                    linked_url = None
                else:
                    linked_url = urljoin_safe(base_url, file_entry.name)
            else:
                linked_url = None

            if linked_url:
                linked_url_info = parse_url_or_log(linked_url)

                if linked_url_info:
                    verdict = self._fetch_rule.check_ftp_request(self._item_session)[0]

                    if verdict:
                        if linked_url_info.path.endswith('/'):
                            self._item_session.add_child_url(linked_url_info.url, link_type=LinkType.directory)
                        else:
                            self._item_session.add_child_url(linked_url_info.url, link_type=LinkType.file, level=level)
1285849
def list(self, url):
        
        url = self.get_full_url(url)
        self.log.debug("list url : " + url)
        # Building request
        request = urllib2.Request(url)
        request.add_header('Content-Type', 'application/json; charset=UTF-8')
        request.add_header('Accept', 'application/json')
        # Do request
        ret = self.do_request(request)
        self.log.debug("""list url : %(url)s : request time : %(time)s""",
                       {"url": url,
                        "time": self.last_req_time})
        return ret
461775
def datascan(self, search: str):
        
        url_path = "/api/datascan/{search}".format(search=search)
        return self._request(path=url_path)
494802
def args_to_search_url(arguments)
      query = if arguments[:query].kind_of? Hash
        #multi-field
        arguments[:query].collect {|field, query_value| fielded_query(query_value, field)}.join(" ")
      elsif arguments[:search_field]
        fielded_query(arguments[:query], arguments[:search_field])
      else
        arguments[:query]
      end

      query_url = base_url + "volumes?q=#{CGI.escape  query}"
      if configuration.api_key
        query_url += "&key=#{configuration.api_key}"
      end

      if arguments[:per_page]
        query_url += "&maxResults=#{arguments[:per_page]}"
      end
      if arguments[:start]
        query_url += "&startIndex=#{arguments[:start]}"
      end

      if arguments[:sort] &&
          (defn = sort_definitions[arguments[:sort]]) &&
          (value = defn[:implementation])
        query_url += "&orderBy=#{CGI.escape(value)}"
      end


      return query_url
    end
1318468
def list(self, request, *args, **kwargs):
        """"""
        search_kwargs = {"published": False}

        query_params = get_query_params(self.request)

        for field_name in ("status"):
            if field_name in query_params:
                search_kwargs[field_name] = query_params.get(field_name)

        if "search" in query_params:
            search_kwargs["query"] = query_params.get("search")

        # active/closed are semantics of the Poll model
        # active is analagous to 'published' in Content
        # closed is in relation to the end_date of a Poll
        if "active" in query_params:
            del search_kwargs["published"]
            search_kwargs["active"] = True

        if "closed" in query_params:
            del search_kwargs["published"]
            search_kwargs["closed"] = True

        queryset = self.model.search_objects.search(**search_kwargs)

        page = self.paginate_queryset(queryset)
        if page is not None:
            serializer = self.get_serializer(page, many=True)
            return self.get_paginated_response(serializer.data)

        serializer = self.get_serializer(queryset, many=True)
        return Response(serializer.data)
905019
def ListHuntApprovals(context=None):
  """"""
  items = context.SendIteratorRequest("ListHuntApprovals",
                                      hunt_pb2.ApiListHuntApprovalsArgs())

  def MapHuntApproval(data):
    return HuntApproval(data=data, username=context.username, context=context)

  return utils.MapItemsIterator(MapHuntApproval, items)
163234
def _build_search_query(self, from_date):
        

        sort = [{self._sort_on_field: {"order": "asc"}}]

        filters = []
        if self._repo:
            filters.append({"term": {"origin": self._repo}})

        if from_date:
            filters.append({"range": {self._sort_on_field: {"gte": from_date}}})

        if filters:
            query = {"bool": {"filter": filters}}
        else:
            query = {"match_all": {}}

        search_query = {
            "query": query,
            "sort": sort
        }

        return search_query
1065637
function extendUrlWithQuery(url, queryArgs) {
  if (_.isEmpty(queryArgs)) {
    return url;
  }

  var parts = urlLib.parse(url);
  var query = _.extend(qs.parse(parts.query), queryArgs);
  parts.search = '?' + qs.stringify(query);
  return urlLib.format(_.pick(
    parts,
    'protocol',
    'slashes',
    'host',
    'auth',
    'pathname',
    'search',
    'hash'
  ));
}
884762
def linkfetch(self):
        "
        request, handle = self.open()
        self._add_headers(request)
        if handle:
            self._get_crawled_urls(handle, request)
747086
def _list_getter(self, uri, headers, last_obj=None, spr=False):
        

        # Quote the file path.
        base_path = marked_path = ('%s?limit=10000&format=json' % uri.path)

        if last_obj:
            marked_path = self._last_marker(
                base_path=base_path,
                last_object=cloud_utils.quoter(last_obj)
            )

        file_list = self._obj_index(
            uri=uri,
            base_path=base_path,
            marked_path=marked_path,
            headers=headers,
            spr=spr
        )

        LOG.debug(
            'Found [ %d ] entries(s) at [ %s ]',
            len(file_list),
            uri.geturl()
        )

        if spr:
            return file_list
        else:
            return cloud_utils.unique_list_dicts(
                dlist=file_list, key='name'
            )