chore: Update data_loader.py with Hugging Face Hub integration

data_loader.py

@@ -3,8 +3,24 @@ from datetime import datetime
 from typing import Any, Dict, List
 
 import pandas as pd
+from dotenv import load_dotenv
+from huggingface_hub import HfApi
+from huggingface_hub.utils import logging
 from tqdm.auto import tqdm
 
+load_dotenv()
+
+HF_TOKEN = os.getenv("HF_TOKEN")
+assert HF_TOKEN is not None, "You need to set HF_TOKEN in your environment variables"
+USER_AGENT = os.getenv("USER_AGENT")
+assert (
+    USER_AGENT is not None
+), "You need to set USER_AGENT in your environment variables"
+
+logger = logging.get_logger(__name__)
+
+api = HfApi(token=HF_TOKEN)
+
 
 def has_card_data(dataset):
     return hasattr(dataset, "card_data")