Sets the whether the spider parses the robots.txt for uris (not related to following the directions).

@param parseRobotsTxt the new value for parseRobotsTxt
353344
public void setParseRobotsTxt(boolean parseRobotsTxt) {
		this.parseRobotsTxt = parseRobotsTxt;
		getConfig().setProperty(SPIDER_PARSE_ROBOTS_TXT, Boolean.toString(parseRobotsTxt));
	}
1367969
public void setTo(String to) {
        try {
            this.to = new URI(to);
        } catch (URISyntaxException e) {
            throw new CitrusRuntimeException("Invalid to uri", e);
        }
    }
1043194
public void setURI(String uri) {
		try {
			this.value = (uri != null) ? new URI(uri) : null;
		} catch (Throwable exception) {
			this.value = null;
		}
		this.type = AttributeType.URI;
		this.assigned = this.value != null;
	}
965919
public void setXssMatchSets(java.util.Collection<XssMatchSetSummary> xssMatchSets) {
        if (xssMatchSets == null) {
            this.xssMatchSets = null;
            return;
        }

        this.xssMatchSets = new java.util.ArrayList<XssMatchSetSummary>(xssMatchSets);
    }
976544
public void setProperty(final String uri, final Object value) throws SAXNotRecognizedException, SAXNotSupportedException {
        reader.setProperty(uri, value);
    }
318931
@Override
    public void setAsText(String text) throws IllegalArgumentException {
        if(text == null){
            setValue(null);
            return;
        }
        setValue(tokenize(text));
    }
1172112
public void setText(int index, String value)
   {
      set(selectField(TaskFieldLists.CUSTOM_TEXT, index), value);
   }
212544
public void setParseComments(boolean parseComments) {
		this.parseComments = parseComments;
		getConfig().setProperty(SPIDER_PARSE_COMMENTS, Boolean.toString(parseComments));
	}
28032
public void setSiteRoot(String siteRoot) {

        if (siteRoot != null) {
            siteRoot = siteRoot.replaceFirst("/$", "");
        }
        m_siteRoot = siteRoot;
    }
591322
public void setAsText(final String text) {
        if (BeanUtils.isNull(text)) {
            setValue(null);
            return;
        }
        try {
            Object newValue = Short.decode(text);
            setValue(newValue);
        } catch (NumberFormatException e) {
            throw new IllegalArgumentException("Failed to parse short.", e);
        }
    }
788241
@Override
    public void setAsText(String text) throws IllegalArgumentException {
        try {
            setValue(new ObjectName(text));
        } catch (MalformedObjectNameException e) {
            throw new IllegalArgumentException(e);
        }

    }
1256882
@Override
    public void setAsText(final String text) {
        if (BeanUtils.isNull(text)){
            setValue(null);
            return;
        }
        String[] tokens = super.tokenize(text);
        char[] theValue = new char[tokens.length];

        for (int index = 0; index < tokens.length; index++) {
            if (tokens[index].length() != 1)
                throw new IllegalArgumentException("Too many (" + tokens[index].length() + ") characters: '" + tokens[index]
                        + "'");
            theValue[index] = tokens[index].charAt(0);
        }
        setValue(theValue);
    }
514988
private void setEditorUri(String uri) {

        if (CmsStringUtil.isEmptyOrWhitespaceOnly(uri)) {
            setValidConfiguration(false);
            LOG.error(Messages.get().getBundle().key(Messages.LOG_EDITOR_CONFIG_NO_URI_0));
        }
        m_editorUri = uri;
    }
226852
@JsonIgnore
  @Override
  public void setUri(URI uri) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException
  {
    super.setUri(uri);
  }
173416
public void setSearchText(String searchText) {
        if (TextUtils.isEmpty(searchText)) {
            searchText = null;
        }
        this.searchText = searchText;
        if (this.searchBox != null) {
            this.searchBox.setText(searchText);
        }
    }
517434
public void setResourceUris(java.util.Collection<ResourceUri> resourceUris) {
        if (resourceUris == null) {
            this.resourceUris = null;
            return;
        }

        this.resourceUris = new java.util.ArrayList<ResourceUri>(resourceUris);
    }
77806
def check_robotstxt(url, session):
    
    roboturl = get_roboturl(url)
    rp = get_robotstxt_parser(roboturl, session=session)
    if not rp.can_fetch(UserAgent, str(url)):
        raise IOError("%s is disallowed by %s" % (url, roboturl))