Crawl image tags found in the HTML page

@param The HTTP response with the crawl options
@param the jquery like object for accessing to the HTML tags.
@param callback()
177098
function crawlImages(result, $, endCallback) {

    if (! result.images) {
      return endCallback();
    }

    log.debug({"url" : result.url, "step" : "analyzeHTML", "message" : "CrawlImages"});

    async.each($('img'), function(img, callback) {
        crawlImage($, result, img, callback);
    }, endCallback);
}
1168835
protected final void registerCustomEventCallback(
            final CrawlEvent event,
            final PatternMatchingCallback callback) {
        Validate.notNull(event, "The event cannot be null.");
        Validate.notNull(callback, "The callback cannot be null.");

        callbackManager.addCustomEventCallback(event, callback);
    }
923371
public void bind(String url, Functions.Func callback) {
        cropper.croppie("bind", url).then((result, object) -> {
            callback.call();
            return true;
        });
    }
862046
function parseUrl(url, callback) {
  var options = {
    uri:            url,
    method:         "GET",
    followRedirect: true
  };

  request(options, function (err, res, body) {
    if (err) {
      callback(err);
    } else {
      callback(null, parse(body));
    }
  });
}
842007
function(callback)
    {
        function mycallback(mimeType, content)
        {
            this._originalContent = content;
            callback(mimeType, content);
        }
        this._uiSourceCode.requestContent(mycallback.bind(this));
    }
1217667
function crawlHref($,result, a, callback) {

      var link = $(a).attr('href');
      var parentUrl = result.url;
      if (link) {

        var anchor = $(a).text() ? $(a).text() : "";
        var noFollow = $(a).attr("rel");
        var isDoFollow =  ! (noFollow && noFollow === "nofollow");

        var linkUri = URI.linkToURI(parentUrl, link);

        pm.crawlLink(parentUrl, linkUri, anchor, isDoFollow, function(){
          addLinkToQueue(result, parentUrl, linkUri, anchor, isDoFollow, callback);
        });

      }
      else {
        callback();
      }

}
542007
@Override
  public void onHttpRequest(
      final ParsedHttpRequest parsedHttpRequest,
      final HttpServletResponse response) throws IOException, ServletException {

    LOG.log(Level.INFO, "HttpServerReefEventHandler in webserver onHttpRequest is called: {0}",
        parsedHttpRequest.getRequestUri());

    final String version = parsedHttpRequest.getVersion().toLowerCase();
    final String target = parsedHttpRequest.getTargetEntity().toLowerCase();

    switch (target) {
    case "evaluators": {
      final String queryStr = parsedHttpRequest.getQueryString();
      if (queryStr == null || queryStr.isEmpty()) {
        if (version.equals(VER)) {
          writeEvaluatorsJsonOutput(response);
        } else {
          writeEvaluatorsWebOutput(response);
        }
      } else {
        handleQueries(response, parsedHttpRequest.getQueryMap(), version);
      }
      break;
    }
    case "driver":
      if (version.equals(VER)) {
        writeDriverJsonInformation(response);
      } else {
        writeDriverWebInformation(response);
      }
      break;
    case "close":
      for (final EventHandler<Void> e : clientCloseHandlers) {
        e.onNext(null);
      }
      response.getWriter().println("Enforced closing");
      break;
    case "kill":
      reefStateManager.onClientKill();
      response.getWriter().println("Killing");
      break;
    case "duration":
      final ArrayList<String> lines =
          LogParser.getFilteredLinesFromFile(driverStderrFile, LoggingScopeImpl.DURATION, LoggingScopeImpl.TOKEN, null);
      writeLines(response, lines, "Performance...");
      break;
    case "stages":
      final ArrayList<String> starts =
          LogParser.getFilteredLinesFromFile(driverStderrFile, LoggingScopeImpl.START_PREFIX, logLevelPrefix, null);
      final ArrayList<String> exits =
          LogParser.getFilteredLinesFromFile(driverStderrFile, LoggingScopeImpl.EXIT_PREFIX, logLevelPrefix,
              LoggingScopeImpl.DURATION);
      final ArrayList<String> startsStages = LogParser.findStages(starts, LogParser.START_INDICATORS);
      final ArrayList<String> endStages = LogParser.findStages(exits, LogParser.END_INDICATORS);
      final ArrayList<String> result = LogParser.mergeStages(startsStages, endStages);
      writeLines(response, result, "Current Stages...");
      break;
    case "logfile":
      final List<String> names = parsedHttpRequest.getQueryMap().get("filename");
      final PrintWriter writer = response.getWriter();
      if (names == null || names.size() == 0) {
        writer.println("File name is not provided");
      } else {
        final String fileName = names.get(0);
        if (!fileName.equals(driverStdoutFile) && !fileName.equals(driverStderrFile)) {
          writer.println(String.format("Unsupported file names: [%s] ", fileName));
        } else {
          try {
            final byte[] outputBody = readFile(fileName).getBytes(StandardCharsets.UTF_8);
            writer.print(Arrays.toString(outputBody));
          } catch (final IOException e) {
            writer.println(String.format("Cannot find the log file: [%s].", fileName));
          }
        }
      }
      break;
    case "progress":
      response.getWriter().println(progressProvider.get().getProgress());
      break;
    default:
      response.getWriter().println(String.format("Unsupported query for entity: [%s].", target));
    }
  }
720403
private void parseResponse(ModelNode response, AsyncCallback<Map<String,String>> callback) {

        //System.out.println(response.toString());

        Map<String, String> serverValues = new HashMap<String,String>();

        if(isStandalone)
        {
            serverValues.put("Standalone Server", response.get(RESULT).asString());
        }
        else if(response.hasDefined("server-groups"))
        {

            List<Property> groups = response.get("server-groups").asPropertyList();
            for(Property serverGroup : groups)
            {
                List<Property> hosts = serverGroup.getValue().get("host").asPropertyList();
                for(Property host : hosts)
                {
                    List<Property> servers = host.getValue().asPropertyList();
                    for(Property server : servers)
                    {
                        serverValues.put(server.getName(),
                                server.getValue().get("response").get("result").asString()
                        );
                    }
                }
            }
        }

        callback.onSuccess(serverValues);
    }
31366
function ( callback ) {
					self.api.call( _.extend( params, continueParams ), function ( err, data, next ) {
						if ( err ) {
							callback( err );
						} else {
							// append batch data
							const batchData = ( typeof key === 'function' ) ? key( data ) : data[ key ];

							res = res.concat( batchData );

							// more pages?
							continueParams = next;
							callback( next ? null : true );
						}
					} );
				}
1200730
function moreLikeThis(uris, callback) {
  // FIXME use all URIs
  var uri = uris[0];
  retrieveByURI(uri, function(err, res) {
    if (err) {
      callback(err);
      return;
    }
    var mlt = require('./moreLikeThis.js');
    var content = utils.getTextFromHtml(res._source.content);

    mlt.fields = sourceFields;
    mlt.query.bool.should[0].mlt_field.content.like_text = content;
    mlt.query.bool.should[1].mlt_field.title.like_text = res._source.title;
    var data = JSON.stringify(mlt);

    doPost('/contentItem/_search', data, callback);
  });
}
580370
public CrawlerResult getUrls(CrawlerConfiguration configuration) {

    final Map<String, String> requestHeaders = configuration.getRequestHeadersMap();
    final HTMLPageResponse resp =
        verifyInput(configuration.getStartUrl(), configuration.getOnlyOnPath(), requestHeaders);

    int level = 0;

    final Set<CrawlerURL> allUrls = new LinkedHashSet<CrawlerURL>();
    final Set<HTMLPageResponse> verifiedUrls = new LinkedHashSet<HTMLPageResponse>();
    final Set<HTMLPageResponse> nonWorkingResponses = new LinkedHashSet<HTMLPageResponse>();

    verifiedUrls.add(resp);

    final String host = resp.getPageUrl().getHost();

    if (configuration.getMaxLevels() > 0) {

      // set the start url
      Set<CrawlerURL> nextToFetch = new LinkedHashSet<CrawlerURL>();
      nextToFetch.add(resp.getPageUrl());

      while (level < configuration.getMaxLevels()) {

        final Map<Future<HTMLPageResponse>, CrawlerURL> futures =
            new HashMap<Future<HTMLPageResponse>, CrawlerURL>(nextToFetch.size());

        for (CrawlerURL testURL : nextToFetch) {
          futures.put(service.submit(new HTMLPageResponseCallable(testURL, responseFetcher, true,
              requestHeaders, false)), testURL);
        }

        nextToFetch =
            fetchNextLevelLinks(futures, allUrls, nonWorkingResponses, verifiedUrls, host,
                configuration.getOnlyOnPath(), configuration.getNotOnPath());
        level++;
      }
    } else {
      allUrls.add(resp.getPageUrl());
    }

    if (configuration.isVerifyUrls())
      verifyUrls(allUrls, verifiedUrls, nonWorkingResponses, requestHeaders);

    LinkedHashSet<CrawlerURL> workingUrls = new LinkedHashSet<CrawlerURL>();
    for (HTMLPageResponse workingResponses : verifiedUrls) {
      workingUrls.add(workingResponses.getPageUrl());
    }
    
    // TODO find a better fix for this
    // wow, this is a hack to fix if the first URL is redirected,
    // then we want to keep that original start url
    if (workingUrls.size() >= 1) {
      List<CrawlerURL> list = new ArrayList<CrawlerURL>(workingUrls);
      list.add(0, new CrawlerURL(configuration.getStartUrl()));
      list.remove(1);
      workingUrls.clear();
      workingUrls.addAll(list);
    }

    return new CrawlerResult(configuration.getStartUrl(), configuration.isVerifyUrls()
        ? workingUrls
        : allUrls, verifiedUrls, nonWorkingResponses);

  }
1161579
@Override
  public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
    JsonGenerator jg = null;
    String jsonpcb = null;
    PrintWriter writer = null;
    try {

      writer = response.getWriter();

      // "callback" parameter implies JSONP output
      jsonpcb = request.getParameter(CALLBACK_PARAM);
      if (jsonpcb != null) {
        response.setContentType("application/javascript; charset=utf8");
        writer.write(jsonpcb + "(");
      } else {
        response.setContentType("application/json; charset=utf8");
      }

      jg = jsonFactory.createJsonGenerator(writer);
      jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
      jg.useDefaultPrettyPrinter();

      int statusCode = renderMBeans(jg, request.getParameterValues("qry"));
      response.setStatus(statusCode);

    } catch (IOException e) {
      writeException(jg, e);
      response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
    } catch (MalformedObjectNameException e) {
      writeException(jg, e);
      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
    } finally {
      if (jg != null) {
        jg.close();
      }
      if (jsonpcb != null) {
        writer.write(");");
      }
      if (writer != null) {
        writer.close();
      }
    }

  }
1059296
function (callback) {
            getData.takeSnapshot(page, ph, function (error, page, ph, dom) {
                if (error) {
                    callback (error);
                    return;
                }

                originalDom = dom;
                callback();
            });
        }
1057551
function (response) {
            response = JSON.parse(response).result;
            response.stdout.forEach(function (val, index, array) {
                if (val == answers[index]) {
                    results.push(true);
                } else {
                    results.push(false);
                }
            });
            callback({
                results: results,
                message: response.message,
                time: response.time
            });
        }
848633
function (options, callback) {
  if (options.html) {
    if (options.url) {
      return callback(true, { error: 'Must specify either `url` or `html`, not both', success: false, requestUrl: options.url, errorDetails: 'Must specify either `url` or `html`, not both' }, null);
    }
    var ogObject = extractMetaTags(options.html, options);
    return callback(false, { data: ogObject, success: true }, null);
  }

  var validate = utils.validate(options.url, options.timeout);

  if (validate.returnInputUrl) {
    options.url = validate.returnInputUrl;
    options.timeout = validate.returnInputTimeout;
    options.headers = Object.assign({
      'user-agent': 'request.js'
    }, options.headers);
    options.gzip = true;
    options.encoding = options.encoding || null;
    options.jar = options.jar || false;
    options.followAllRedirects = options.followAllRedirects || true;
    options.maxRedirects = options.maxRedirects || 20;

    if (process.browser) {
      options.gzip = false;
      options.protocol = url.parse(options.url).protocol;
    }

    // trying to limit non html pages
    if (utils.endsWith(validate.returnInputUrl, '.jpg') ||
      utils.endsWith(validate.returnInputUrl, '.jpeg') ||
      utils.endsWith(validate.returnInputUrl, '.png') ||
      utils.endsWith(validate.returnInputUrl, '.zip') ||
      utils.endsWith(validate.returnInputUrl, '.pdf')) {
      return callback(true, { error: 'Must scrape an HTML page', success: false, requestUrl: options.url, errorDetails: 'Must scrape an HTML page' }, null);
    }

    // see if site is black listed
    if (options.blacklist && options.blacklist.length > 0) {
      for (var i = 0; i < options.blacklist.length; i++) {
        if (options.url.indexOf(options.blacklist[i]) !== -1) {
          return callback(true, { error: 'Host Name Has Been Black Listed', success: false, requestUrl: options.url, errorDetails: 'Host Name Has Been Black Listed' }, null);
        }
      }
    }

    return requestAndResultsFormatter(options, function (error, results, response) {
      if (results) {
        return callback(false, { data: results, success: true, requestUrl: options.url }, response);
      }
      if (error && (error.code === 'ENOTFOUND' || error.code === 'EHOSTUNREACH')) {
        return callback(true, { error: 'Page Not Found', success: false, requestUrl: options.url, errorDetails: error, response: response }, response);
      } else if (error && error.code === 'ETIMEDOUT') {
        return callback(true, { error: 'Time Out', success: false, requestUrl: options.url, errorDetails: error, response: response }, response);
      }
      return callback(true, { error: 'Page Not Found', success: false, requestUrl: options.url, errorDetails: error, response: response }, response);
    });
  }

  return callback(true, { error: 'Invalid URL', success: false, requestUrl: options.url, errorDetails: 'Invalid URL' }, null);
}
551754
function eachImg(layer, callback) {
      var imgtags = layer._container.children[0].children;
      for (var i in imgtags) {
        if (imgtags[i].tagName === 'IMG') {
          callback(imgtags[i]);
        }
      }
    }
124897
function(p, callback) {

		var _this = this;
		var error = _this.error;

		// This has to go through a POST request
		if (!_this.isEmpty(p.data) && !('FileList' in window) && _this.hasBinary(p.data)) {

			// Disable XHR and JSONP
			p.xhr = false;
			p.jsonp = false;
		}

		// Check if the browser and service support CORS
		var cors = this.request_cors(function() {
			// If it does then run this...
			return ((p.xhr === undefined) || (p.xhr && (typeof (p.xhr) !== 'function' || p.xhr(p, p.query))));
		});

		if (cors) {

			formatUrl(p, function(url) {

				var x = _this.xhr(p.method, url, p.headers, p.data, callback);
				x.onprogress = p.onprogress || null;

				// Windows Phone does not support xhr.upload, see #74
				// Feature detect
				if (x.upload && p.onuploadprogress) {
					x.upload.onprogress = p.onuploadprogress;
				}

			});

			return;
		}

		// Clone the query object
		// Each request modifies the query object and needs to be tared after each one.
		var _query = p.query;

		p.query = _this.clone(p.query);

		// Assign a new callbackID
		p.callbackID = _this.globalEvent();

		// JSONP
		if (p.jsonp !== false) {

			// Clone the query object
			p.query.callback = p.callbackID;

			// If the JSONP is a function then run it
			if (typeof (p.jsonp) === 'function') {
				p.jsonp(p, p.query);
			}

			// Lets use JSONP if the method is 'get'
			if (p.method === 'get') {

				formatUrl(p, function(url) {
					_this.jsonp(url, callback, p.callbackID, p.timeout);
				});

				return;
			}
			else {
				// It's not compatible reset query
				p.query = _query;
			}

		}

		// Otherwise we're on to the old school, iframe hacks and JSONP
		if (p.form !== false) {

			// Add some additional query parameters to the URL
			// We're pretty stuffed if the endpoint doesn't like these
			p.query.redirect_uri = p.redirect_uri;
			p.query.state = JSON.stringify({callback:p.callbackID});

			var opts;

			if (typeof (p.form) === 'function') {

				// Format the request
				opts = p.form(p, p.query);
			}

			if (p.method === 'post' && opts !== false) {

				formatUrl(p, function(url) {
					_this.post(url, p.data, opts, callback, p.callbackID, p.timeout);
				});

				return;
			}
		}

		// None of the methods were successful throw an error
		callback(error('invalid_request', 'There was no mechanism for handling this request'));

		return;

		// Format URL
		// Constructs the request URL, optionally wraps the URL through a call to a proxy server
		// Returns the formatted URL
		function formatUrl(p, callback) {

			// Are we signing the request?
			var sign;

			// OAuth1
			// Remove the token from the query before signing
			if (p.authResponse && p.authResponse.oauth && parseInt(p.authResponse.oauth.version, 10) === 1) {

				// OAUTH SIGNING PROXY
				sign = p.query.access_token;

				// Remove the access_token
				delete p.query.access_token;

				// Enfore use of Proxy
				p.proxy = true;
			}

			// POST body to querystring
			if (p.data && (p.method === 'get' || p.method === 'delete')) {
				// Attach the p.data to the querystring.
				_this.extend(p.query, p.data);
				p.data = null;
			}

			// Construct the path
			var path = _this.qs(p.url, p.query);

			// Proxy the request through a server
			// Used for signing OAuth1
			// And circumventing services without Access-Control Headers
			if (p.proxy) {
				// Use the proxy as a path
				path = _this.qs(p.oauth_proxy, {
					path: path,
					access_token: sign || '',

					// This will prompt the request to be signed as though it is OAuth1
					then: p.proxy_response_type || (p.method.toLowerCase() === 'get' ? 'redirect' : 'proxy'),
					method: p.method.toLowerCase(),
					suppress_response_codes: true
				});
			}

			callback(path);
		}
	}