switch to wikipedia loader because of duckduckgo anti-bot protection

requirements.txt

@@ -10,3 +10,5 @@ langchain-openai
 langchain-anthropic
 langgraph
 certifi
+arxiv
+pymupdf

tools.py

@@ -8,6 +8,7 @@ import pandas as pd
 from openai import OpenAI
 from pytubefix import YouTube
 from langchain_community.tools import tool
+from langchain_community.document_loaders import WikipediaLoader
 from bs4 import BeautifulSoup, ResultSet, PageElement, Tag, NavigableString
 
 urllib3.disable_warnings()
@@ -84,22 +85,11 @@ def wiki_search(query: str) -> str | None:
     """
 
     try:
-        ddg_results = []
         wiki_results = ""
-        link_rows = _fetch_ddg_search_result_links(f"wikipedia {query}")
-        print(query, link_rows)
-        for link_row in link_rows:
-            if not 'en.wikipedia.org' in link_row.attrs['href']:
-                continue
-
-            ddg_results.append({
-                'title': link_row.get_text(strip=True),
-                'url': link_row.attrs['href']
-            })
-
-            wiki_results += _fetch_specific_page(link_row.attrs['href'])
-            if len(ddg_results) == 1:
-                break
+        search_docs = WikipediaLoader(query=query, load_max_docs=1).load()
+        for doc in search_docs:
+            if "source" in doc.metadata and doc.metadata["source"]:
+                wiki_results += _fetch_specific_page(doc.metadata["source"])
 
         return wiki_results
     except requests.exceptions.RequestException as e:
@@ -121,7 +111,6 @@ def archive_search(query: str) -> str | None:
         ddg_results = []
         archive_results = ""
         link_rows = _fetch_ddg_search_result_links(f"archive.org {query}")
-        print(query, link_rows)
         for link_row in link_rows:
             if not 'archive.org' in link_row.attrs['href']:
                 continue
@@ -268,7 +257,6 @@ def _fetch_ddg_search_result_links(query: str) -> ResultSet[PageElement | Tag |
 
     ddg_response = requests.get(url, headers=headers, params=params, verify=False)
     ddg_response.raise_for_status()
-    print(ddg_response.text)
     soup = BeautifulSoup(ddg_response.text, 'html.parser')
     return soup.find_all('a', {'class': 'result-link'})