e

<span class="code-comment"># Function to check robots.txt</span>
def is_allowed(url, user_agent="*"):
    import urllib.robotparser
    
    <span class="code-comment"># Parse the URL to get the base domain</span>
    from urllib.parse import urlparse
    parsed_url = urlparse(url)
    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
    
    <span class="code-comment"># Initialize the RobotFileParser</span>
    rp = urllib.robotparser.RobotFileParser()
    rp.set_url(f"{base_url}/robots.txt")
   