 
    try:
        rp.read()
        return rp.can_fetch(user_agent, url)
    except:
        <span class="code-comment"># If robots.txt can't be read, assume scraping is allowed</span>
        return True

<span class="code-comment"># Example usage</span>
url_to_scrape = "https://example.com/products"

if is_allowed(url_to_scrape, session.headers['User-Agent']):
    soup = scrape_page(url_to_scrape)
    if soup:
        <span class="code-comment"># Process the page</span>
        print("Successfully scrape