use full hf comp code

competitions

@@ -1 +0,0 @@
-Subproject commit 447fff164c586ae3d42617c7726d8563514c3f17

competitions/__init__.py

@@ -0,0 +1,6 @@
+import os
+
+
+__version__ = "0.1.9.dev0"
+
+HF_URL = os.getenv("HF_URL", "https://huggingface.co")

competitions/api.py

@@ -0,0 +1,109 @@
+import asyncio
+import os
+import signal
+import sqlite3
+from contextlib import asynccontextmanager
+
+import psutil
+from fastapi import FastAPI
+from loguru import logger
+
+from competitions.utils import run_evaluation
+
+
+def get_process_status(pid):
+    try:
+        process = psutil.Process(pid)
+        proc_status = process.status()
+        return proc_status
+    except psutil.NoSuchProcess:
+        logger.info(f"No process found with PID: {pid}")
+        return "Completed"
+
+
+def kill_process_by_pid(pid):
+    """Kill process by PID."""
+    os.kill(pid, signal.SIGTERM)
+
+
+class JobDB:
+    def __init__(self, db_path):
+        self.db_path = db_path
+        self.conn = sqlite3.connect(db_path)
+        self.c = self.conn.cursor()
+        self.create_jobs_table()
+
+    def create_jobs_table(self):
+        self.c.execute(
+            """CREATE TABLE IF NOT EXISTS jobs
+            (id INTEGER PRIMARY KEY, pid INTEGER)"""
+        )
+        self.conn.commit()
+
+    def add_job(self, pid):
+        sql = f"INSERT INTO jobs (pid) VALUES ({pid})"
+        self.c.execute(sql)
+        self.conn.commit()
+
+    def get_running_jobs(self):
+        self.c.execute("""SELECT pid FROM jobs""")
+        running_pids = self.c.fetchall()
+        running_pids = [pid[0] for pid in running_pids]
+        return running_pids
+
+    def delete_job(self, pid):
+        sql = f"DELETE FROM jobs WHERE pid={pid}"
+        self.c.execute(sql)
+        self.conn.commit()
+
+
+PARAMS = os.environ.get("PARAMS")
+DB = JobDB("job.db")
+
+
+class BackgroundRunner:
+    async def run_main(self):
+        while True:
+            running_jobs = DB.get_running_jobs()
+            if running_jobs:
+                for _pid in running_jobs:
+                    proc_status = get_process_status(_pid)
+                    proc_status = proc_status.strip().lower()
+                    if proc_status in ("completed", "error", "zombie"):
+                        logger.info(f"Process {_pid} is already completed. Skipping...")
+                        try:
+                            kill_process_by_pid(_pid)
+                        except Exception as e:
+                            logger.info(f"Error while killing process: {e}")
+                        DB.delete_job(_pid)
+
+            running_jobs = DB.get_running_jobs()
+            if not running_jobs:
+                logger.info("No running jobs found. Shutting down the server.")
+                os.kill(os.getpid(), signal.SIGINT)
+            await asyncio.sleep(30)
+
+
+runner = BackgroundRunner()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    process_pid = run_evaluation(params=PARAMS)
+    logger.info(f"Started training with PID {process_pid}")
+    DB.add_job(process_pid)
+    asyncio.create_task(runner.run_main())
+    yield
+
+
+api = FastAPI(lifespan=lifespan)
+
+
+@api.get("/")
+async def root():
+    return "Your model is being evaluated..."
+
+
+@api.get("/health")
+async def health():
+    return "OK"

competitions/app.py

@@ -0,0 +1,447 @@
+import datetime
+import os
+import threading
+import time
+
+from fastapi import Depends, FastAPI, File, Form, HTTPException, Request, UploadFile
+from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+from huggingface_hub import hf_hub_download
+from huggingface_hub.utils import disable_progress_bars
+from huggingface_hub.utils._errors import EntryNotFoundError
+from loguru import logger
+from pydantic import BaseModel
+from requests.exceptions import RequestException
+
+from competitions import __version__, utils
+from competitions.errors import AuthenticationError, PastDeadlineError, SubmissionError, SubmissionLimitError
+from competitions.info import CompetitionInfo
+from competitions.leaderboard import Leaderboard
+from competitions.oauth import attach_oauth
+from competitions.runner import JobRunner
+from competitions.submissions import Submissions
+from competitions.text import SUBMISSION_SELECTION_TEXT, SUBMISSION_TEXT
+
+
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+COMPETITION_ID = os.environ.get("COMPETITION_ID")
+OUTPUT_PATH = os.environ.get("OUTPUT_PATH", "/tmp/model")
+START_DATE = os.environ.get("START_DATE", "2000-12-31")
+DISABLE_PUBLIC_LB = int(os.environ.get("DISABLE_PUBLIC_LB", 0))
+
+disable_progress_bars()
+
+try:
+    REQUIREMENTS_FNAME = hf_hub_download(
+        repo_id=COMPETITION_ID,
+        filename="requirements.txt",
+        token=HF_TOKEN,
+        repo_type="dataset",
+    )
+except EntryNotFoundError:
+    REQUIREMENTS_FNAME = None
+
+if REQUIREMENTS_FNAME:
+    logger.info("Uninstalling and installing requirements")
+    utils.uninstall_requirements(REQUIREMENTS_FNAME)
+    utils.install_requirements(REQUIREMENTS_FNAME)
+
+
+class LeaderboardRequest(BaseModel):
+    lb: str
+
+
+class UpdateSelectedSubmissionsRequest(BaseModel):
+    submission_ids: str
+
+
+class UpdateTeamNameRequest(BaseModel):
+    new_team_name: str
+
+
+def run_job_runner():
+    job_runner = JobRunner(
+        competition_id=COMPETITION_ID,
+        token=HF_TOKEN,
+        output_path=OUTPUT_PATH,
+    )
+    job_runner.run()
+
+
+def start_job_runner_thread():
+    thread = threading.Thread(target=run_job_runner)
+    # thread.daemon = True
+    thread.start()
+    return thread
+
+
+def watchdog(job_runner_thread):
+    while True:
+        if not job_runner_thread.is_alive():
+            logger.warning("Job runner thread stopped. Restarting...")
+            job_runner_thread = start_job_runner_thread()
+        time.sleep(10)
+
+
+job_runner_thread = start_job_runner_thread()
+watchdog_thread = threading.Thread(target=watchdog, args=(job_runner_thread,))
+watchdog_thread.daemon = True
+watchdog_thread.start()
+
+
+app = FastAPI()
+attach_oauth(app)
+
+static_path = os.path.join(BASE_DIR, "static")
+app.mount("/static", StaticFiles(directory=static_path), name="static")
+templates_path = os.path.join(BASE_DIR, "templates")
+templates = Jinja2Templates(directory=templates_path)
+
+
+@app.get("/", response_class=HTMLResponse)
+async def read_form(request: Request):
+    """
+    This function is used to render the HTML file
+    :param request:
+    :return:
+    """
+    if HF_TOKEN is None:
+        return HTTPException(status_code=500, detail="HF_TOKEN is not set.")
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    context = {
+        "request": request,
+        "logo": competition_info.logo_url,
+        "competition_type": competition_info.competition_type,
+        "version": __version__,
+        "rules_available": competition_info.rules is not None,
+    }
+    return templates.TemplateResponse("index.html", context)
+
+
+@app.get("/login_status", response_class=JSONResponse)
+async def use_oauth(request: Request, user_token: str = Depends(utils.user_authentication)):
+    if user_token:
+        return {"response": 2}
+    return {"response": 1}
+
+
+@app.get("/logout", response_class=HTMLResponse)
+async def user_logout(request: Request):
+    """Endpoint that logs out the user (e.g. delete cookie session)."""
+
+    if "oauth_info" in request.session:
+        request.session.pop("oauth_info", None)
+
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    context = {
+        "request": request,
+        "logo": competition_info.logo_url,
+        "competition_type": competition_info.competition_type,
+        "__version__": __version__,
+        "rules_available": competition_info.rules is not None,
+    }
+
+    return templates.TemplateResponse("index.html", context)
+
+
+@app.get("/competition_info", response_class=JSONResponse)
+async def get_comp_info(request: Request):
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    info = competition_info.competition_desc
+    resp = {"response": info}
+    return resp
+
+
+@app.get("/dataset_info", response_class=JSONResponse)
+async def get_dataset_info(request: Request):
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    info = competition_info.dataset_desc
+    resp = {"response": info}
+    return resp
+
+
+@app.get("/rules", response_class=JSONResponse)
+async def get_rules(request: Request):
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    if competition_info.rules is not None:
+        return {"response": competition_info.rules}
+    return {"response": "No rules available."}
+
+
+@app.get("/submission_info", response_class=JSONResponse)
+async def get_submission_info(request: Request):
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    info = competition_info.submission_desc
+    resp = {"response": info}
+    return resp
+
+
+@app.post("/leaderboard", response_class=JSONResponse)
+async def fetch_leaderboard(
+    request: Request, body: LeaderboardRequest, user_token: str = Depends(utils.user_authentication)
+):
+    lb = body.lb
+
+    comp_org = COMPETITION_ID.split("/")[0]
+    if user_token is not None:
+        is_user_admin = utils.is_user_admin(user_token, comp_org)
+    else:
+        is_user_admin = False
+
+    if DISABLE_PUBLIC_LB == 1 and lb == "public" and not is_user_admin:
+        return {"response": "Public leaderboard is disabled by the competition host."}
+
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    leaderboard = Leaderboard(
+        end_date=competition_info.end_date,
+        eval_higher_is_better=competition_info.eval_higher_is_better,
+        max_selected_submissions=competition_info.selection_limit,
+        competition_id=COMPETITION_ID,
+        token=HF_TOKEN,
+        scoring_metric=competition_info.scoring_metric,
+    )
+    if lb == "private":
+        current_utc_time = datetime.datetime.now()
+        if current_utc_time < competition_info.end_date and not is_user_admin:
+            return {"response": f"Private leaderboard will be available on {competition_info.end_date} UTC."}
+    df = leaderboard.fetch(private=lb == "private")
+
+    if len(df) == 0:
+        return {"response": "No teams yet. Why not make a submission?"}
+    resp = {"response": df.to_markdown(index=False)}
+    return resp
+
+
+@app.post("/my_submissions", response_class=JSONResponse)
+async def my_submissions(request: Request, user_token: str = Depends(utils.user_authentication)):
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    if user_token is None:
+        return {
+            "response": {
+                "submissions": "",
+                "submission_text": SUBMISSION_TEXT.format(competition_info.submission_limit),
+                "error": "**Invalid token. Please login.**",
+                "team_name": "",
+            }
+        }
+    sub = Submissions(
+        end_date=competition_info.end_date,
+        submission_limit=competition_info.submission_limit,
+        competition_id=COMPETITION_ID,
+        token=HF_TOKEN,
+        competition_type=competition_info.competition_type,
+        hardware=competition_info.hardware,
+    )
+    try:
+        subs = sub.my_submissions(user_token)
+    except AuthenticationError:
+        return {
+            "response": {
+                "submissions": "",
+                "submission_text": SUBMISSION_TEXT.format(competition_info.submission_limit),
+                "error": "**Invalid token. Please login.**",
+                "team_name": "",
+            }
+        }
+    subs = subs.to_dict(orient="records")
+    error = ""
+    if len(subs) == 0:
+        error = "**You have not made any submissions yet.**"
+        subs = ""
+    submission_text = SUBMISSION_TEXT.format(competition_info.submission_limit)
+    submission_selection_text = SUBMISSION_SELECTION_TEXT.format(competition_info.selection_limit)
+
+    team_name = utils.get_team_name(user_token, COMPETITION_ID, HF_TOKEN)
+
+    resp = {
+        "response": {
+            "submissions": subs,
+            "submission_text": submission_text + submission_selection_text,
+            "error": error,
+            "team_name": team_name,
+        }
+    }
+    return resp
+
+
+@app.post("/new_submission", response_class=JSONResponse)
+async def new_submission(
+    request: Request,
+    submission_file: UploadFile = File(None),
+    hub_model: str = Form(...),
+    submission_comment: str = Form(None),
+    user_token: str = Depends(utils.user_authentication),
+):
+    if submission_comment is None:
+        submission_comment = ""
+
+    if user_token is None:
+        return {"response": "Invalid token. Please login."}
+
+    todays_date = datetime.datetime.now()
+    start_date = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
+    if todays_date < start_date:
+        comp_org = COMPETITION_ID.split("/")[0]
+        if not utils.is_user_admin(user_token, comp_org):
+            return {"response": "Competition has not started yet!"}
+
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    sub = Submissions(
+        end_date=competition_info.end_date,
+        submission_limit=competition_info.submission_limit,
+        competition_id=COMPETITION_ID,
+        token=HF_TOKEN,
+        competition_type=competition_info.competition_type,
+        hardware=competition_info.hardware,
+    )
+    try:
+        if competition_info.competition_type == "generic":
+            resp = sub.new_submission(user_token, submission_file, submission_comment)
+            return {"response": f"Success! You have {resp} submissions remaining today."}
+        if competition_info.competition_type == "script":
+            resp = sub.new_submission(user_token, hub_model, submission_comment)
+            return {"response": f"Success! You have {resp} submissions remaining today."}
+    except RequestException:
+        return {"response": "Hugging Face Hub is unreachable, please try again later"}
+    except AuthenticationError:
+        return {"response": "Invalid token"}
+    except PastDeadlineError:
+        return {"response": "Competition has ended"}
+    except SubmissionError:
+        return {"response": "Invalid submission file"}
+    except SubmissionLimitError:
+        return {"response": "Submission limit reached"}
+    return {"response": "Invalid competition type"}
+
+
+@app.post("/update_selected_submissions", response_class=JSONResponse)
+def update_selected_submissions(
+    request: Request, body: UpdateSelectedSubmissionsRequest, user_token: str = Depends(utils.user_authentication)
+):
+    submission_ids = body.submission_ids
+
+    if user_token is None:
+        return {"success": False, "error": "Invalid token, please login."}
+
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+    sub = Submissions(
+        end_date=competition_info.end_date,
+        submission_limit=competition_info.submission_limit,
+        competition_id=COMPETITION_ID,
+        token=HF_TOKEN,
+        competition_type=competition_info.competition_type,
+        hardware=competition_info.hardware,
+    )
+    submission_ids = submission_ids.split(",")
+    submission_ids = [s.strip() for s in submission_ids]
+    if len(submission_ids) > competition_info.selection_limit:
+        return {
+            "success": False,
+            "error": f"Please select at most {competition_info.selection_limit} submissions.",
+        }
+    sub.update_selected_submissions(user_token=user_token, selected_submission_ids=submission_ids)
+    return {"success": True, "error": ""}
+
+
+@app.post("/update_team_name", response_class=JSONResponse)
+def update_team_name(
+    request: Request, body: UpdateTeamNameRequest, user_token: str = Depends(utils.user_authentication)
+):
+    new_team_name = body.new_team_name
+
+    if user_token is None:
+        return {"success": False, "error": "Invalid token. Please login."}
+
+    if str(new_team_name).strip() == "":
+        return {"success": False, "error": "Team name cannot be empty."}
+
+    try:
+        utils.update_team_name(user_token, new_team_name, COMPETITION_ID, HF_TOKEN)
+        return {"success": True, "error": ""}
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+
+@app.post("/admin/comp_info", response_class=JSONResponse)
+async def admin_comp_info(request: Request, user_token: str = Depends(utils.user_authentication)):
+    comp_org = COMPETITION_ID.split("/")[0]
+    user_is_admin = utils.is_user_admin(user_token, comp_org)
+    if not user_is_admin:
+        return {"response": "You are not an admin."}, 403
+
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+
+    markdowns = {
+        "competition_desc": competition_info.competition_desc,
+        "rules": competition_info.rules,
+        "submission_desc": competition_info.submission_desc,
+        "dataset_desc": competition_info.dataset_desc,
+    }
+    if markdowns["rules"] is None:
+        markdowns["rules"] = "No rules available."
+
+    config = {
+        "SUBMISSION_LIMIT": competition_info.submission_limit,
+        "SELECTION_LIMIT": competition_info.selection_limit,
+        "END_DATE": competition_info.end_date.strftime("%Y-%m-%d"),
+        "EVAL_HIGHER_IS_BETTER": competition_info.eval_higher_is_better,
+        "SUBMISSION_COLUMNS": competition_info.submission_columns_raw,
+        "SUBMISSION_ID_COLUMN": competition_info.submission_id_col,
+        "LOGO": competition_info.logo_url,
+        "COMPETITION_TYPE": competition_info.competition_type,
+        "EVAL_METRIC": competition_info.metric,
+        "SUBMISSION_ROWS": competition_info.submission_rows,
+        "TIME_LIMIT": competition_info.time_limit,
+        "DATASET": competition_info.dataset,
+        "SUBMISSION_FILENAMES": competition_info.submission_filenames,
+        "SCORING_METRIC": competition_info.scoring_metric,
+        "HARDWARE": competition_info.hardware,
+    }
+
+    return {"response": {"config": config, "markdowns": markdowns}}
+
+
+@app.post("/admin/update_comp_info", response_class=JSONResponse)
+async def update_comp_info(request: Request, user_token: str = Depends(utils.user_authentication)):
+    comp_org = COMPETITION_ID.split("/")[0]
+    user_is_admin = utils.is_user_admin(user_token, comp_org)
+    if not user_is_admin:
+        return {"response": "You are not an admin."}, 403
+
+    competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
+
+    data = await request.json()
+    config = data["config"]
+    markdowns = data["markdowns"]
+
+    valid_keys = [
+        "SUBMISSION_LIMIT",
+        "SELECTION_LIMIT",
+        "END_DATE",
+        "EVAL_HIGHER_IS_BETTER",
+        "SUBMISSION_COLUMNS",
+        "SUBMISSION_ID_COLUMN",
+        "LOGO",
+        "COMPETITION_TYPE",
+        "EVAL_METRIC",
+        "SUBMISSION_ROWS",
+        "TIME_LIMIT",
+        "DATASET",
+        "SUBMISSION_FILENAMES",
+        "SCORING_METRIC",
+        "HARDWARE",
+    ]
+
+    for key in config:
+        if key not in valid_keys:
+            return {"success": False, "error": f"Invalid key: {key}"}
+
+    try:
+        competition_info.update_competition_info(config, markdowns, HF_TOKEN)
+    except Exception as e:
+        logger.error(e)
+        return {"success": False}, 500
+
+    return {"success": True}

competitions/cli/__init__.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from argparse import ArgumentParser
+
+
+class BaseCompetitionsCommand(ABC):
+    @staticmethod
+    @abstractmethod
+    def register_subcommand(parser: ArgumentParser):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def run(self):
+        raise NotImplementedError()

competitions/cli/competitions.py

@@ -0,0 +1,38 @@
+import argparse
+
+from .. import __version__
+from .create import CreateCompetitionAppCommand
+from .run import RunCompetitionsAppCommand
+from .submit import SubmitCompetitionAppCommand
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        "Competitions CLI",
+        usage="competitions <command> [<args>]",
+        epilog="For more information about a command, run: `competitions <command> --help`",
+    )
+    parser.add_argument("--version", "-v", help="Display competitions version", action="store_true")
+    commands_parser = parser.add_subparsers(help="commands")
+
+    # Register commands
+    RunCompetitionsAppCommand.register_subcommand(commands_parser)
+    CreateCompetitionAppCommand.register_subcommand(commands_parser)
+    SubmitCompetitionAppCommand.register_subcommand(commands_parser)
+
+    args = parser.parse_args()
+
+    if args.version:
+        print(__version__)
+        exit(0)
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        exit(1)
+
+    command = args.func(args)
+    command.run()
+
+
+if __name__ == "__main__":
+    main()

competitions/cli/create.py

@@ -0,0 +1,20 @@
+from argparse import ArgumentParser
+
+from . import BaseCompetitionsCommand
+
+
+def create_command_factory(args):
+    return CreateCompetitionAppCommand()
+
+
+class CreateCompetitionAppCommand(BaseCompetitionsCommand):
+    @staticmethod
+    def register_subcommand(parser: ArgumentParser):
+        create_project_parser = parser.add_parser("create", description="✨ Start UI to create a new competition")
+        create_project_parser.set_defaults(func=create_command_factory)
+
+    def run(self):
+        from competitions.create import main
+
+        demo = main()
+        demo.launch()

competitions/cli/run.py

@@ -0,0 +1,27 @@
+from argparse import ArgumentParser
+
+from . import BaseCompetitionsCommand
+
+
+def run_app_command_factory(args):
+    return RunCompetitionsAppCommand(args)
+
+
+class RunCompetitionsAppCommand(BaseCompetitionsCommand):
+    @staticmethod
+    def register_subcommand(parser: ArgumentParser):
+        create_project_parser = parser.add_parser("run", description="✨ Run competitions app")
+        create_project_parser.add_argument("--host", default="0.0.0.0", help="Host to run app on")
+        create_project_parser.add_argument("--port", default=7860, help="Port to run app on")
+        create_project_parser.set_defaults(func=run_app_command_factory)
+
+    def __init__(self, args):
+        self.host = args.host
+        self.port = args.port
+
+    def run(self):
+        import uvicorn
+
+        from competitions.app import app
+
+        uvicorn.run(app, host=self.host, port=self.port)

competitions/cli/submit.py

@@ -0,0 +1,49 @@
+import os
+from argparse import ArgumentParser
+
+import requests
+
+from . import BaseCompetitionsCommand
+
+
+def submit_commands_factory(args):
+    return SubmitCompetitionAppCommand(args)
+
+
+class SubmitCompetitionAppCommand(BaseCompetitionsCommand):
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def register_subcommand(parser: ArgumentParser):
+        submit_competition_parser = parser.add_parser("submit", description="Submit to a competition")
+        submit_competition_parser.add_argument(
+            "--competition_id", type=str, help="ID of the competition, e.g. huggingface/cool-competition"
+        )
+        submit_competition_parser.add_argument(
+            "--submission", type=str, help="Path to submission file or HuggingFace hub repo"
+        )
+        submit_competition_parser.add_argument("--comment", type=str, help="Submission comment", default="")
+        submit_competition_parser.add_argument("--token", type=str, help="User token, read-only", default="")
+        submit_competition_parser.set_defaults(func=submit_commands_factory)
+
+    def run(self):
+        if os.path.isfile(self.args.submission):
+            files = {"submission_file": open(self.args.submission, "rb")}
+            data = {
+                "hub_model": "None",
+                "submission_comment": self.args.comment,
+            }
+        else:
+            files = {"submission_file": None}
+            data = {
+                "hub_model": self.args.submission,
+                "submission_comment": self.args.comment,
+            }
+
+        headers = {"Authorization": f"Bearer {self.args.token}"}
+
+        api_url = "https://" + self.args.competition_id.replace("/", "-") + ".hf.space/new_submission"
+
+        response = requests.post(api_url, data=data, files=files, headers=headers)
+        print(response.json())

competitions/compute_metrics.py

@@ -0,0 +1,73 @@
+import importlib
+import os
+import sys
+
+import pandas as pd
+from huggingface_hub import hf_hub_download
+from sklearn import metrics
+
+
+def compute_metrics(params):
+    if params.metric == "custom":
+        metric_file = hf_hub_download(
+            repo_id=params.competition_id,
+            filename="metric.py",
+            token=params.token,
+            repo_type="dataset",
+        )
+        sys.path.append(os.path.dirname(metric_file))
+        metric = importlib.import_module("metric")
+        evaluation = metric.compute(params)
+    else:
+        solution_file = hf_hub_download(
+            repo_id=params.competition_id,
+            filename="solution.csv",
+            token=params.token,
+            repo_type="dataset",
+        )
+
+        solution_df = pd.read_csv(solution_file)
+
+        submission_filename = f"submissions/{params.team_id}-{params.submission_id}.csv"
+        submission_file = hf_hub_download(
+            repo_id=params.competition_id,
+            filename=submission_filename,
+            token=params.token,
+            repo_type="dataset",
+        )
+        submission_df = pd.read_csv(submission_file)
+
+        public_ids = solution_df[solution_df.split == "public"][params.submission_id_col].values
+        private_ids = solution_df[solution_df.split == "private"][params.submission_id_col].values
+
+        public_solution_df = solution_df[solution_df[params.submission_id_col].isin(public_ids)]
+        public_submission_df = submission_df[submission_df[params.submission_id_col].isin(public_ids)]
+
+        private_solution_df = solution_df[solution_df[params.submission_id_col].isin(private_ids)]
+        private_submission_df = submission_df[submission_df[params.submission_id_col].isin(private_ids)]
+
+        public_solution_df = public_solution_df.sort_values(params.submission_id_col).reset_index(drop=True)
+        public_submission_df = public_submission_df.sort_values(params.submission_id_col).reset_index(drop=True)
+
+        private_solution_df = private_solution_df.sort_values(params.submission_id_col).reset_index(drop=True)
+        private_submission_df = private_submission_df.sort_values(params.submission_id_col).reset_index(drop=True)
+
+        _metric = getattr(metrics, params.metric)
+        target_cols = [col for col in solution_df.columns if col not in [params.submission_id_col, "split"]]
+        public_score = _metric(public_solution_df[target_cols], public_submission_df[target_cols])
+        private_score = _metric(private_solution_df[target_cols], private_submission_df[target_cols])
+
+        # scores can also be dictionaries for multiple metrics
+        evaluation = {
+            "public_score": {
+                params.metric: public_score,
+            },
+            "private_score": {
+                params.metric: private_score,
+            },
+        }
+
+    # check all keys in public_score and private_score are same
+    if evaluation["public_score"].keys() != evaluation["private_score"].keys():
+        raise ValueError("Public and private scores have different keys")
+    return evaluation

competitions/create.py

@@ -0,0 +1,348 @@
+import io
+import json
+
+import gradio as gr
+from huggingface_hub import HfApi
+from loguru import logger
+
+from competitions.utils import token_information
+
+
+COMPETITION_DESC = """Sample competition description"""
+DATASET_DESC = """Sample dataset description"""
+SUBMISSION_DESC = """Sample submission description"""
+RULES = """Sample rules"""
+SOLUTION_CSV = """
+id,pred,split
+0,1,public
+1,0,private
+2,0,private
+3,1,private
+4,0,public
+5,1,private
+6,1,public
+7,1,private
+8,0,public
+9,0,private
+10,0,private
+11,0,private
+12,1,private
+13,0,private
+14,1,public
+15,1,private
+16,1,private
+17,0,private
+18,0,private
+19,0,public
+20,0,private
+21,0,private
+22,1,private
+23,1,public
+24,0,private
+25,0,private
+26,0,public
+27,1,private
+28,1,private
+29,0,private
+30,0,public
+"""
+SOLUTION_CSV = SOLUTION_CSV.strip()
+
+DOCKERFILE = """
+FROM huggingface/competitions:latest
+
+CMD uvicorn competitions.app:app --host 0.0.0.0 --port 7860 --workers 1
+"""
+DOCKERFILE = DOCKERFILE.replace("\n", " ").replace("  ", "\n").strip()
+
+HARDWARE_CHOICES = [
+    "cpu-basic",
+    "cpu-upgrade",
+    "t4-small",
+    "t4-medium",
+    "a10g-small",
+    "a10g-large",
+    "a10g-largex2",
+    "a10g-largex4",
+    "a100-large",
+]
+METRIC_CHOICES = [
+    "accuracy_score",
+    "f1_score",
+    "hamming_loss",
+    "jaccard_score",
+    "log_loss",
+    "roc_auc_score",
+    "mean_squared_error",
+    "mean_absolute_error",
+    "r2_score",
+    "custom",
+]
+
+
+def check_if_user_can_create_competition(user_token):
+    """
+    Check if the user can create a competition
+    :param user_token: the user's token
+    :return: True if the user can create a competition, False otherwise
+    """
+    user_info = token_information(user_token)
+    valid_orgs = user_info["orgs"]
+
+    return gr.Dropdown(
+        choices=valid_orgs,
+        visible=True,
+        value=valid_orgs[0],
+    )
+
+
+def _create_readme(competition_name):
+    _readme = "---\n"
+    _readme += f"title: {competition_name}\n"
+    _readme += "emoji: 🚀\n"
+    _readme += "colorFrom: green\n"
+    _readme += "colorTo: indigo\n"
+    _readme += "sdk: docker\n"
+    _readme += "pinned: false\n"
+    _readme += "tags:\n"
+    _readme += "  - competition\n"
+    _readme += "hf_oauth: true\n"
+    _readme += "hf_oauth_scopes:\n"
+    _readme += "  - read-repos\n"
+    _readme += "---\n"
+    _readme = io.BytesIO(_readme.encode())
+    return _readme
+
+
+def _create(
+    user_token,
+    organization,
+    competition_name,
+    competition_logo,
+    hardware,
+    competition_type,
+    time_limit,
+    metric,
+    metric_higher_is_better,
+    submission_limit,
+    selection_limit,
+    end_date,
+    submission_id_column,
+    submission_columns,
+    submission_rows,
+):
+    """
+    Create a competition
+    """
+
+    # make sure competition name is alphanumeric
+    competition_name = "".join([c for c in competition_name if c.isalnum()])
+    if len(competition_name) == 0:
+        raise gr.Error("Please provide a valid alphanumeric competition name")
+
+    conf_json = {
+        "COMPETITION_TYPE": competition_type,
+        "SUBMISSION_LIMIT": int(submission_limit),
+        "TIME_LIMIT": int(time_limit),
+        "SELECTION_LIMIT": int(selection_limit),
+        "HARDWARE": hardware,
+        "END_DATE": end_date,
+        "EVAL_HIGHER_IS_BETTER": metric_higher_is_better is True,
+        "SUBMISSION_ID_COLUMN": submission_id_column,
+        "SUBMISSION_COLUMNS": submission_columns,
+        "SUBMISSION_ROWS": int(submission_rows),
+        "EVAL_METRIC": metric,
+        "LOGO": competition_logo,
+        "DATASET": "",
+        "SUBMISSION_FILENAMES": ["submission.csv"],
+        "SCORING_METRIC": "",
+    }
+    teams_json = {}
+    user_team_json = {}
+
+    logger.info(f"Creating competition: {competition_name}")
+
+    api = HfApi(token=user_token)
+    api.create_repo(
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+        private=True,
+    )
+
+    conf_json = json.dumps(conf_json, indent=4)
+    conf_json_bytes = conf_json.encode("utf-8")
+    conf_json_buffer = io.BytesIO(conf_json_bytes)
+    api.upload_file(
+        path_or_fileobj=conf_json_buffer,
+        path_in_repo="conf.json",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+    )
+
+    teams_json = json.dumps(teams_json, indent=4)
+    teams_json_bytes = teams_json.encode("utf-8")
+    teams_json_buffer = io.BytesIO(teams_json_bytes)
+    api.upload_file(
+        path_or_fileobj=teams_json_buffer,
+        path_in_repo="teams.json",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+    )
+
+    user_team_json = json.dumps(user_team_json, indent=4)
+    user_team_json_bytes = user_team_json.encode("utf-8")
+    user_team_json_buffer = io.BytesIO(user_team_json_bytes)
+    api.upload_file(
+        path_or_fileobj=user_team_json_buffer,
+        path_in_repo="user_team.json",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+    )
+
+    comp_desc = io.BytesIO(COMPETITION_DESC.encode())
+    api.upload_file(
+        path_or_fileobj=comp_desc,
+        path_in_repo="COMPETITION_DESC.md",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+    )
+
+    dataset_desc = io.BytesIO(DATASET_DESC.encode())
+    api.upload_file(
+        path_or_fileobj=dataset_desc,
+        path_in_repo="DATASET_DESC.md",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+    )
+
+    submission_desc = io.BytesIO(SUBMISSION_DESC.encode())
+    api.upload_file(
+        path_or_fileobj=submission_desc,
+        path_in_repo="SUBMISSION_DESC.md",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+    )
+
+    solution_csv = io.BytesIO(SOLUTION_CSV.encode())
+    api.upload_file(
+        path_or_fileobj=solution_csv,
+        path_in_repo="solution.csv",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+    )
+
+    rules = io.BytesIO(RULES.encode())
+    api.upload_file(
+        path_or_fileobj=rules,
+        path_in_repo="RULES.md",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="dataset",
+    )
+
+    # create competition space
+    api.create_repo(
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="space",
+        space_sdk="docker",
+        space_hardware="cpu-basic" if competition_type == "script" else hardware,
+        private=True,
+    )
+    api.add_space_secret(repo_id=f"{organization}/{competition_name}", key="HF_TOKEN", value=user_token)
+    api.add_space_secret(
+        repo_id=f"{organization}/{competition_name}",
+        key="COMPETITION_ID",
+        value=f"{organization}/{competition_name}",
+    )
+    readme = _create_readme(competition_name)
+    api.upload_file(
+        path_or_fileobj=readme,
+        path_in_repo="README.md",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="space",
+    )
+
+    _dockerfile = io.BytesIO(DOCKERFILE.encode())
+    api.upload_file(
+        path_or_fileobj=_dockerfile,
+        path_in_repo="Dockerfile",
+        repo_id=f"{organization}/{competition_name}",
+        repo_type="space",
+    )
+
+    return gr.Markdown(
+        value=f"""Created private dataset and competition space.
+        To make competition public, you should make the space public.
+        Please note that the dataset should always be kept private.
+
+        Private dataset: https://huggingface.co/datasets/{organization}/{competition_name}
+
+        Competition space: https://huggingface.co/spaces/{organization}/{competition_name}
+
+        Note: there's still some work left. Now you must change the solution.csv file to your own solution,
+        and make changes to *_desc.md files to reflect your competition. You may also change conf.json
+        to suit your needs. Please refer to the [documentation](https://hf.co/docs/competitions) for more information.
+        """
+    )
+
+
+def main():
+    with gr.Blocks() as demo:
+        gr.Markdown("# Hugging Face Competition Creator")
+        token = gr.Textbox(label="Your Hugging Face write token", lines=1, type="password")
+        with gr.Row():
+            organization = gr.Dropdown(label="Organization name", choices=[""])
+            competition_name = gr.Textbox(label="Competition name", lines=1)
+            competition_logo = gr.Textbox(label="Competition logo", value="https://mysite.com/mylogo.png", lines=1)
+        with gr.Group():
+            with gr.Row():
+                hardware = gr.Dropdown(label="Hardware to use", choices=HARDWARE_CHOICES, value=HARDWARE_CHOICES[0])
+                competition_type = gr.Dropdown(
+                    label="Competition type", choices=["generic", "script"], value="generic"
+                )
+                time_limit = gr.Textbox(
+                    label="Time limit (s). Only used for script competitions", lines=1, value="3600"
+                )
+            with gr.Row():
+                metric = gr.Dropdown(label="Metric to use", choices=METRIC_CHOICES, value=METRIC_CHOICES[0])
+                metric_higher_is_better = gr.Dropdown(
+                    label="Is higher metric better?", choices=[True, False], value=True
+                )
+            with gr.Row():
+                submission_limit = gr.Textbox(label="Submission limit per day", lines=1, value="5")
+                selection_limit = gr.Textbox(label="Final selection limit", lines=1, value="2")
+                end_date = gr.Textbox(label="End date (YYYY-MM-DD)", lines=1, value="2024-12-31")
+            with gr.Row():
+                submission_id_column = gr.Textbox(label="Submission id column", lines=1, value="id")
+                submission_columns = gr.Textbox(label="Submission columns", lines=1, value="id,pred")
+                submission_rows = gr.Textbox(label="Submission total rows (exclusing header)", lines=1, value="10000")
+
+        output_md = gr.Markdown("Click the button below to create the competition")
+        create_competition = gr.Button(value="Create competition")
+        token.change(check_if_user_can_create_competition, inputs=token, outputs=organization)
+
+        create_competition.click(
+            _create,
+            inputs=[
+                token,
+                organization,
+                competition_name,
+                competition_logo,
+                hardware,
+                competition_type,
+                time_limit,
+                metric,
+                metric_higher_is_better,
+                submission_limit,
+                selection_limit,
+                end_date,
+                submission_id_column,
+                submission_columns,
+                submission_rows,
+            ],
+            outputs=output_md,
+        )
+    return demo
+
+
+if __name__ == "__main__":
+    main().launch()

competitions/download.py

@@ -0,0 +1,206 @@
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+from huggingface_hub import HfApi
+from huggingface_hub.constants import DEFAULT_REVISION, HUGGINGFACE_HUB_CACHE, REPO_TYPES
+from huggingface_hub.file_download import REGEX_COMMIT_HASH, hf_hub_download, repo_folder_name
+from huggingface_hub.utils import filter_repo_objects, validate_hf_hub_args
+from joblib import Parallel, delayed
+
+
+@validate_hf_hub_args
+def snapshot_download(
+    repo_id: str,
+    *,
+    revision: Optional[str] = None,
+    repo_type: Optional[str] = None,
+    cache_dir: Union[str, Path, None] = None,
+    library_name: Optional[str] = None,
+    library_version: Optional[str] = None,
+    user_agent: Optional[Union[Dict, str]] = None,
+    proxies: Optional[Dict] = None,
+    etag_timeout: Optional[float] = 10,
+    resume_download: Optional[bool] = False,
+    use_auth_token: Optional[Union[bool, str]] = None,
+    local_files_only: Optional[bool] = False,
+    allow_regex: Optional[Union[List[str], str]] = None,
+    ignore_regex: Optional[Union[List[str], str]] = None,
+    allow_patterns: Optional[Union[List[str], str]] = None,
+    ignore_patterns: Optional[Union[List[str], str]] = None,
+) -> str:
+    """Download all files of a repo.
+
+    Downloads a whole snapshot of a repo's files at the specified revision. This
+    is useful when you want all files from a repo, because you don't know which
+    ones you will need a priori. All files are nested inside a folder in order
+    to keep their actual filename relative to that folder.
+
+    An alternative would be to just clone a repo but this would require that the
+    user always has git and git-lfs installed, and properly configured.
+
+    Args:
+        repo_id (`str`):
+            A user or an organization name and a repo name separated by a `/`.
+        revision (`str`, *optional*):
+            An optional Git revision id which can be a branch name, a tag, or a
+            commit hash.
+        repo_type (`str`, *optional*):
+            Set to `"dataset"` or `"space"` if uploading to a dataset or space,
+            `None` or `"model"` if uploading to a model. Default is `None`.
+        cache_dir (`str`, `Path`, *optional*):
+            Path to the folder where cached files are stored.
+        library_name (`str`, *optional*):
+            The name of the library to which the object corresponds.
+        library_version (`str`, *optional*):
+            The version of the library.
+        user_agent (`str`, `dict`, *optional*):
+            The user-agent info in the form of a dictionary or a string.
+        proxies (`dict`, *optional*):
+            Dictionary mapping protocol to the URL of the proxy passed to
+            `requests.request`.
+        etag_timeout (`float`, *optional*, defaults to `10`):
+            When fetching ETag, how many seconds to wait for the server to send
+            data before giving up which is passed to `requests.request`.
+        resume_download (`bool`, *optional*, defaults to `False):
+            If `True`, resume a previously interrupted download.
+        use_auth_token (`str`, `bool`, *optional*):
+            A token to be used for the download.
+                - If `True`, the token is read from the HuggingFace config
+                  folder.
+                - If a string, it's used as the authentication token.
+        local_files_only (`bool`, *optional*, defaults to `False`):
+            If `True`, avoid downloading the file and return the path to the
+            local cached file if it exists.
+        allow_patterns (`List[str]` or `str`, *optional*):
+            If provided, only files matching at least one pattern are downloaded.
+        ignore_patterns (`List[str]` or `str`, *optional*):
+            If provided, files matching any of the patterns are not downloaded.
+
+    Returns:
+        Local folder path (string) of repo snapshot
+
+    <Tip>
+
+    Raises the following errors:
+
+    - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
+      if `use_auth_token=True` and the token cannot be found.
+    - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
+      ETag cannot be determined.
+    - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+      if some parameter value is invalid
+
+    </Tip>
+    """
+    if cache_dir is None:
+        cache_dir = HUGGINGFACE_HUB_CACHE
+    if revision is None:
+        revision = DEFAULT_REVISION
+    if isinstance(cache_dir, Path):
+        cache_dir = str(cache_dir)
+
+    if repo_type is None:
+        repo_type = "model"
+    if repo_type not in REPO_TYPES:
+        raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are:" f" {str(REPO_TYPES)}")
+
+    storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
+
+    # TODO: remove these 4 lines in version 0.12
+    #       Deprecated code to ensure backward compatibility.
+    if allow_regex is not None:
+        allow_patterns = allow_regex
+    if ignore_regex is not None:
+        ignore_patterns = ignore_regex
+
+    # if we have no internet connection we will look for an
+    # appropriate folder in the cache
+    # If the specified revision is a commit hash, look inside "snapshots".
+    # If the specified revision is a branch or tag, look inside "refs".
+    if local_files_only:
+        if REGEX_COMMIT_HASH.match(revision):
+            commit_hash = revision
+        else:
+            # retrieve commit_hash from file
+            ref_path = os.path.join(storage_folder, "refs", revision)
+            with open(ref_path) as f:
+                commit_hash = f.read()
+
+        snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash)
+
+        if os.path.exists(snapshot_folder):
+            return snapshot_folder
+
+        raise ValueError(
+            "Cannot find an appropriate cached snapshot folder for the specified"
+            " revision on the local disk and outgoing traffic has been disabled. To"
+            " enable repo look-ups and downloads online, set 'local_files_only' to"
+            " False."
+        )
+
+    # if we have internet connection we retrieve the correct folder name from the huggingface api
+    _api = HfApi()
+    repo_info = _api.repo_info(
+        repo_id=repo_id,
+        repo_type=repo_type,
+        revision=revision,
+        use_auth_token=use_auth_token,
+    )
+    filtered_repo_files = list(
+        filter_repo_objects(
+            items=[f.rfilename for f in repo_info.siblings],
+            allow_patterns=allow_patterns,
+            ignore_patterns=ignore_patterns,
+        )
+    )
+    commit_hash = repo_info.sha
+    snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash)
+    # if passed revision is not identical to commit_hash
+    # then revision has to be a branch name or tag name.
+    # In that case store a ref.
+    if revision != commit_hash:
+        ref_path = os.path.join(storage_folder, "refs", revision)
+        os.makedirs(os.path.dirname(ref_path), exist_ok=True)
+        with open(ref_path, "w") as f:
+            f.write(commit_hash)
+
+    # we pass the commit_hash to hf_hub_download
+    # so no network call happens if we already
+    # have the file locally.
+
+    # for repo_file in tqdm(filtered_repo_files, f"Fetching {len(filtered_repo_files)} files"):
+    #     _ = hf_hub_download(
+    #         repo_id,
+    #         filename=repo_file,
+    #         repo_type=repo_type,
+    #         revision=commit_hash,
+    #         cache_dir=cache_dir,
+    #         library_name=library_name,
+    #         library_version=library_version,
+    #         user_agent=user_agent,
+    #         proxies=proxies,
+    #         etag_timeout=etag_timeout,
+    #         resume_download=resume_download,
+    #         use_auth_token=use_auth_token,
+    #     )
+
+    Parallel(n_jobs=10, backend="threading")(
+        delayed(hf_hub_download)(
+            repo_id,
+            filename=repo_file,
+            repo_type=repo_type,
+            revision=commit_hash,
+            cache_dir=cache_dir,
+            library_name=library_name,
+            library_version=library_version,
+            user_agent=user_agent,
+            proxies=proxies,
+            etag_timeout=etag_timeout,
+            resume_download=resume_download,
+            use_auth_token=use_auth_token,
+        )
+        for repo_file in filtered_repo_files
+    )
+
+    return snapshot_folder

competitions/enums.py

@@ -0,0 +1,14 @@
+import enum
+
+
+class SubmissionStatus(enum.Enum):
+    PENDING = 0
+    QUEUED = 1
+    PROCESSING = 2
+    SUCCESS = 3
+    FAILED = 4
+
+
+class CompetitionType(enum.Enum):
+    GENERIC = 1
+    SCRIPT = 2

competitions/errors.py

@@ -0,0 +1,18 @@
+class AuthenticationError(Exception):
+    pass
+
+
+class NoSubmissionError(Exception):
+    pass
+
+
+class SubmissionError(Exception):
+    pass
+
+
+class SubmissionLimitError(Exception):
+    pass
+
+
+class PastDeadlineError(Exception):
+    pass

competitions/evaluate.py

@@ -0,0 +1,135 @@
+import argparse
+import json
+import os
+import shlex
+import shutil
+import subprocess
+
+from huggingface_hub import HfApi, hf_hub_download, snapshot_download
+from huggingface_hub.utils._errors import EntryNotFoundError
+from loguru import logger
+
+from competitions import utils
+from competitions.compute_metrics import compute_metrics
+from competitions.enums import SubmissionStatus
+from competitions.params import EvalParams
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", type=str, required=True)
+    return parser.parse_args()
+
+
+def upload_submission_file(params, file_path):
+    logger.info("Uploading submission file")
+    pass
+
+
+def generate_submission_file(params):
+    logger.info("Downloading submission dataset")
+    submission_dir = snapshot_download(
+        repo_id=params.submission_repo,
+        local_dir=params.output_path,
+        token=os.environ.get("USER_TOKEN"),
+        repo_type="model",
+    )
+    # submission_dir has a script.py file
+    # start a subprocess to run the script.py
+    # the script.py will generate a submission.csv file in the submission_dir
+    # push the submission.csv file to the repo using upload_submission_file
+    logger.info("Generating submission file")
+
+    # invalidate USER_TOKEN env var
+    os.environ["USER_TOKEN"] = ""
+
+    # Copy sandbox to submission_dir
+    shutil.copyfile("sandbox", f"{submission_dir}/sandbox")
+    sandbox_path = f"{submission_dir}/sandbox"
+    os.chmod(sandbox_path, 0o755)
+    os.chown(sandbox_path, os.getuid(), os.getgid())
+
+    # Define your command
+    cmd = f"{sandbox_path} python script.py"
+    cmd = shlex.split(cmd)
+
+    # Copy the current environment and modify it
+    env = os.environ.copy()
+
+    # Start the subprocess
+    process = subprocess.Popen(cmd, cwd=submission_dir, env=env)
+
+    # Wait for the process to complete or timeout
+    try:
+        process.wait(timeout=params.time_limit)
+    except subprocess.TimeoutExpired:
+        logger.info(f"Process exceeded {params.time_limit} seconds time limit. Terminating...")
+        process.kill()
+        process.wait()
+
+    # Check if process terminated due to timeout
+    if process.returncode and process.returncode != 0:
+        logger.error("Subprocess didn't terminate successfully")
+    else:
+        logger.info("Subprocess terminated successfully")
+
+    logger.info("contents of submission_dir")
+    logger.info(os.listdir(submission_dir))
+
+    api = HfApi(token=params.token)
+    for sub_file in params.submission_filenames:
+        logger.info(f"Uploading {sub_file} to the repository")
+        sub_file_ext = sub_file.split(".")[-1]
+        api.upload_file(
+            path_or_fileobj=f"{submission_dir}/{sub_file}",
+            path_in_repo=f"submissions/{params.team_id}-{params.submission_id}.{sub_file_ext}",
+            repo_id=params.competition_id,
+            repo_type="dataset",
+        )
+
+
+@utils.monitor
+def run(params):
+    logger.info(params)
+    if isinstance(params, dict):
+        params = EvalParams(**params)
+
+    utils.update_submission_status(params, SubmissionStatus.PROCESSING.value)
+
+    if params.competition_type == "script":
+        try:
+            requirements_fname = hf_hub_download(
+                repo_id=params.competition_id,
+                filename="requirements.txt",
+                token=params.token,
+                repo_type="dataset",
+            )
+        except EntryNotFoundError:
+            requirements_fname = None
+
+        if requirements_fname:
+            logger.info("Installing requirements")
+            utils.uninstall_requirements(requirements_fname)
+            utils.install_requirements(requirements_fname)
+        if len(str(params.dataset).strip()) > 0:
+            # _ = Repository(local_dir="/tmp/data", clone_from=params.dataset, token=params.token)
+            _ = snapshot_download(
+                repo_id=params.dataset,
+                local_dir="/tmp/data",
+                token=params.token,
+                repo_type="dataset",
+            )
+        generate_submission_file(params)
+
+    evaluation = compute_metrics(params)
+
+    utils.update_submission_score(params, evaluation["public_score"], evaluation["private_score"])
+    utils.update_submission_status(params, SubmissionStatus.SUCCESS.value)
+    utils.delete_space(params)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    _params = json.load(open(args.config, encoding="utf-8"))
+    _params = EvalParams(**_params)
+    run(_params)

competitions/info.py

@@ -0,0 +1,197 @@
+import io
+import json
+from dataclasses import dataclass
+from datetime import datetime
+
+from huggingface_hub import HfApi, hf_hub_download
+
+
+@dataclass
+class CompetitionInfo:
+    competition_id: str
+    autotrain_token: str
+
+    def __post_init__(self):
+        config_fname = hf_hub_download(
+            repo_id=self.competition_id,
+            filename="conf.json",
+            use_auth_token=self.autotrain_token,
+            repo_type="dataset",
+        )
+        competition_desc = hf_hub_download(
+            repo_id=self.competition_id,
+            filename="COMPETITION_DESC.md",
+            use_auth_token=self.autotrain_token,
+            repo_type="dataset",
+        )
+        dataset_desc = hf_hub_download(
+            repo_id=self.competition_id,
+            filename="DATASET_DESC.md",
+            use_auth_token=self.autotrain_token,
+            repo_type="dataset",
+        )
+        self.config = self.load_config(config_fname)
+        self.competition_desc = self.load_md(competition_desc)
+        self.dataset_desc = self.load_md(dataset_desc)
+        try:
+            submission_desc = hf_hub_download(
+                repo_id=self.competition_id,
+                filename="SUBMISSION_DESC.md",
+                use_auth_token=self.autotrain_token,
+                repo_type="dataset",
+            )
+            self.submission_desc = self.load_md(submission_desc)
+        except Exception:
+            self.submission_desc = None
+
+        try:
+            rules_md = hf_hub_download(
+                repo_id=self.competition_id,
+                filename="RULES.md",
+                use_auth_token=self.autotrain_token,
+                repo_type="dataset",
+            )
+            self.rules_md = self.load_md(rules_md)
+        except Exception:
+            self.rules_md = None
+
+        if self.config["EVAL_METRIC"] == "custom":
+            if "SCORING_METRIC" not in self.config:
+                raise ValueError(
+                    "For custom metrics, please provide a single SCORING_METRIC name in the competition config file: conf.json"
+                )
+
+    def load_md(self, md_path):
+        with open(md_path, "r", encoding="utf-8") as f:
+            md = f.read()
+        return md
+
+    def load_config(self, config_path):
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = json.load(f)
+        return config
+
+    @property
+    def submission_limit(self):
+        return self.config["SUBMISSION_LIMIT"]
+
+    @property
+    def selection_limit(self):
+        return self.config["SELECTION_LIMIT"]
+
+    @property
+    def end_date(self):
+        e_d = self.config["END_DATE"]
+        return datetime.strptime(e_d, "%Y-%m-%d")
+
+    @property
+    def eval_higher_is_better(self):
+        hb = self.config["EVAL_HIGHER_IS_BETTER"]
+        return True if int(hb) == 1 else False
+
+    @property
+    def competition_description(self):
+        return self.competition_desc
+
+    @property
+    def submission_columns(self):
+        return self.config["SUBMISSION_COLUMNS"].split(",")
+
+    @property
+    def submission_columns_raw(self):
+        return self.config["SUBMISSION_COLUMNS"]
+
+    @property
+    def submission_description(self):
+        return self.submission_desc
+
+    @property
+    def dataset_description(self):
+        return self.dataset_desc
+
+    @property
+    def logo_url(self):
+        return self.config["LOGO"]
+
+    @property
+    def competition_type(self):
+        return self.config["COMPETITION_TYPE"].lower().strip()
+
+    @property
+    def metric(self):
+        return self.config["EVAL_METRIC"]
+
+    @property
+    def submission_id_col(self):
+        return self.config["SUBMISSION_ID_COLUMN"]
+
+    @property
+    def submission_cols(self):
+        cols = self.config["SUBMISSION_COLUMNS"].split(",")
+        cols = [c.strip() for c in cols]
+        return cols
+
+    @property
+    def submission_rows(self):
+        return self.config["SUBMISSION_ROWS"]
+
+    @property
+    def time_limit(self):
+        return self.config["TIME_LIMIT"]
+
+    @property
+    def hardware(self):
+        return self.config.get("HARDWARE", "cpu-basic")
+
+    @property
+    def dataset(self):
+        return self.config.get("DATASET", "")
+
+    @property
+    def submission_filenames(self):
+        return self.config.get("SUBMISSION_FILENAMES", ["submission.csv"])
+
+    @property
+    def scoring_metric(self):
+        if self.config["EVAL_METRIC"] == "custom":
+            if "SCORING_METRIC" not in self.config:
+                raise Exception("Please provide a single SCORING_METRIC in the competition config file: conf.json")
+            if self.config["SCORING_METRIC"] is None:
+                raise Exception("Please provide a single SCORING_METRIC in the competition config file: conf.json")
+            return self.config["SCORING_METRIC"]
+        return self.config["EVAL_METRIC"]
+
+    @property
+    def rules(self):
+        return self.rules_md
+
+    def _save_md(self, md, filename, api):
+        md = io.BytesIO(md.encode())
+        api.upload_file(
+            path_or_fileobj=md,
+            path_in_repo=filename,
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+
+    def update_competition_info(self, config, markdowns, token):
+        api = HfApi(token=token)
+        conf_json = json.dumps(config, indent=4)
+        conf_json_bytes = conf_json.encode("utf-8")
+        conf_json_buffer = io.BytesIO(conf_json_bytes)
+        api.upload_file(
+            path_or_fileobj=conf_json_buffer,
+            path_in_repo="conf.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+
+        competition_desc = markdowns["competition_desc"]
+        dataset_desc = markdowns["dataset_desc"]
+        submission_desc = markdowns["submission_desc"]
+        rules_md = markdowns["rules"]
+
+        self._save_md(competition_desc, "COMPETITION_DESC.md", api)
+        self._save_md(dataset_desc, "DATASET_DESC.md", api)
+        self._save_md(submission_desc, "SUBMISSION_DESC.md", api)
+        self._save_md(rules_md, "RULES.md", api)

competitions/leaderboard.py

@@ -0,0 +1,230 @@
+import glob
+import json
+import os
+import time
+from dataclasses import dataclass
+from datetime import datetime
+
+import pandas as pd
+from huggingface_hub import hf_hub_download, snapshot_download
+from loguru import logger
+
+from competitions.enums import SubmissionStatus
+
+
+@dataclass
+class Leaderboard:
+    end_date: datetime
+    eval_higher_is_better: bool
+    max_selected_submissions: int
+    competition_id: str
+    token: str
+    scoring_metric: str
+
+    def __post_init__(self):
+        self.non_score_columns = ["id", "submission_datetime"]
+
+    def _process_public_lb(self):
+        start_time = time.time()
+        submissions_folder = snapshot_download(
+            repo_id=self.competition_id,
+            allow_patterns="submission_info/*.json",
+            use_auth_token=self.token,
+            repo_type="dataset",
+        )
+        logger.info(f"Downloaded submissions in {time.time() - start_time} seconds")
+        start_time = time.time()
+        submissions = []
+        for submission in glob.glob(os.path.join(submissions_folder, "submission_info", "*.json")):
+            with open(submission, "r", encoding="utf-8") as f:
+                submission_info = json.load(f)
+            # only select submissions that are done
+            submission_info["submissions"] = [
+                sub for sub in submission_info["submissions"] if sub["status"] == SubmissionStatus.SUCCESS.value
+            ]
+            submission_info["submissions"] = [
+                sub
+                for sub in submission_info["submissions"]
+                if datetime.strptime(sub["datetime"], "%Y-%m-%d %H:%M:%S") < self.end_date
+            ]
+            if len(submission_info["submissions"]) == 0:
+                continue
+
+            user_id = submission_info["id"]
+            user_submissions = []
+            for sub in submission_info["submissions"]:
+                _sub = {
+                    "id": user_id,
+                    # "submission_id": sub["submission_id"],
+                    # "submission_comment": sub["submission_comment"],
+                    # "status": sub["status"],
+                    # "selected": sub["selected"],
+                }
+                for k, v in sub["public_score"].items():
+                    _sub[k] = v
+                _sub["submission_datetime"] = sub["datetime"]
+                user_submissions.append(_sub)
+
+            user_submissions.sort(key=lambda x: x[self.scoring_metric], reverse=self.eval_higher_is_better)
+            best_user_submission = user_submissions[0]
+            submissions.append(best_user_submission)
+        logger.info(f"Processed submissions in {time.time() - start_time} seconds")
+        return submissions
+
+    def _process_private_lb(self):
+        start_time = time.time()
+        submissions_folder = snapshot_download(
+            repo_id=self.competition_id,
+            allow_patterns="submission_info/*.json",
+            use_auth_token=self.token,
+            repo_type="dataset",
+        )
+        logger.info(f"Downloaded submissions in {time.time() - start_time} seconds")
+        start_time = time.time()
+        submissions = []
+        for submission in glob.glob(os.path.join(submissions_folder, "submission_info", "*.json")):
+            with open(submission, "r", encoding="utf-8") as f:
+                submission_info = json.load(f)
+                submission_info["submissions"] = [
+                    sub for sub in submission_info["submissions"] if sub["status"] == SubmissionStatus.SUCCESS.value
+                ]
+                if len(submission_info["submissions"]) == 0:
+                    continue
+
+                user_id = submission_info["id"]
+                user_submissions = []
+                for sub in submission_info["submissions"]:
+                    _sub = {
+                        "id": user_id,
+                        # "submission_id": sub["submission_id"],
+                        # "submission_comment": sub["submission_comment"],
+                        # "status": sub["status"],
+                        "selected": sub["selected"],
+                    }
+                    for k, v in sub["public_score"].items():
+                        _sub[f"public_{k}"] = v
+                    for k, v in sub["private_score"].items():
+                        _sub[f"private_{k}"] = v
+                    _sub["submission_datetime"] = sub["datetime"]
+                    user_submissions.append(_sub)
+
+                # count the number of submissions which are selected
+                selected_submissions = 0
+                for sub in user_submissions:
+                    if sub["selected"]:
+                        selected_submissions += 1
+
+                if selected_submissions == 0:
+                    # select submissions with best public score
+                    user_submissions.sort(
+                        key=lambda x: x[f"public_{self.scoring_metric}"], reverse=self.eval_higher_is_better
+                    )
+                    # select only the best submission
+                    best_user_submission = user_submissions[0]
+
+                elif selected_submissions <= self.max_selected_submissions:
+                    # select only the selected submissions
+                    user_submissions = [sub for sub in user_submissions if sub["selected"]]
+                    # sort by private score
+                    user_submissions.sort(
+                        key=lambda x: x[f"private_{self.scoring_metric}"], reverse=self.eval_higher_is_better
+                    )
+                    # select only the best submission
+                    best_user_submission = user_submissions[0]
+                else:
+                    logger.warning(
+                        f"User {user_id} has more than {self.max_selected_submissions} selected submissions. Skipping user..."
+                    )
+                    continue
+
+                # remove all keys that start with "public_"
+                best_user_submission = {k: v for k, v in best_user_submission.items() if not k.startswith("public_")}
+
+                # remove private_ from the keys
+                best_user_submission = {k.replace("private_", ""): v for k, v in best_user_submission.items()}
+
+                # remove selected key
+                best_user_submission.pop("selected")
+                submissions.append(best_user_submission)
+        logger.info(f"Processed submissions in {time.time() - start_time} seconds")
+        return submissions
+
+    def fetch(self, private=False):
+        if private:
+            submissions = self._process_private_lb()
+        else:
+            submissions = self._process_public_lb()
+
+        if len(submissions) == 0:
+            return pd.DataFrame()
+
+        df = pd.DataFrame(submissions)
+
+        # convert submission datetime to pandas datetime
+        df["submission_datetime"] = pd.to_datetime(df["submission_datetime"], format="%Y-%m-%d %H:%M:%S")
+
+        # only keep submissions before the end date
+        df = df[df["submission_datetime"] < self.end_date].reset_index(drop=True)
+
+        # sort by submission datetime
+        # sort by public score and submission datetime
+        if self.eval_higher_is_better:
+            if private:
+                df = df.sort_values(
+                    by=[self.scoring_metric, "submission_datetime"],
+                    ascending=[False, True],
+                )
+            else:
+                df = df.sort_values(
+                    by=[self.scoring_metric, "submission_datetime"],
+                    ascending=[False, True],
+                )
+        else:
+            if private:
+                df = df.sort_values(
+                    by=[self.scoring_metric, "submission_datetime"],
+                    ascending=[True, True],
+                )
+            else:
+                df = df.sort_values(
+                    by=[self.scoring_metric, "submission_datetime"],
+                    ascending=[True, True],
+                )
+
+        # only keep 4 significant digits in the scores
+        for col in df.columns:
+            if col in self.non_score_columns:
+                continue
+            df[col] = df[col].round(4)
+
+        # reset index
+        df = df.reset_index(drop=True)
+        df["rank"] = df.index + 1
+
+        # convert datetime column to string
+        df["submission_datetime"] = df["submission_datetime"].dt.strftime("%Y-%m-%d %H:%M:%S")
+
+        # send submission_datetime to the end
+        columns = df.columns.tolist()
+        columns.remove("submission_datetime")
+        columns.append("submission_datetime")
+        df = df[columns]
+
+        # send rank to first position
+        columns = df.columns.tolist()
+        columns.remove("rank")
+        columns = ["rank"] + columns
+        df = df[columns]
+
+        team_metadata = hf_hub_download(
+            repo_id=self.competition_id,
+            filename="teams.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        with open(team_metadata, "r", encoding="utf-8") as f:
+            team_metadata = json.load(f)
+
+        df["id"] = df["id"].apply(lambda x: team_metadata[x]["name"])
+
+        return df

competitions/oauth.py

@@ -0,0 +1,122 @@
+"""OAuth support for AutoTrain.
+Taken from: https://github.com/gradio-app/gradio/blob/main/gradio/oauth.py
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import urllib.parse
+
+import fastapi
+from authlib.integrations.base_client.errors import MismatchingStateError
+from authlib.integrations.starlette_client import OAuth
+from fastapi.responses import RedirectResponse
+from starlette.middleware.sessions import SessionMiddleware
+
+
+OAUTH_CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID")
+OAUTH_CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET")
+OAUTH_SCOPES = os.environ.get("OAUTH_SCOPES")
+OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL")
+
+
+def attach_oauth(app: fastapi.FastAPI):
+    if os.environ.get("USER_TOKEN") is not None:
+        return
+    _add_oauth_routes(app)
+    # Session Middleware requires a secret key to sign the cookies. Let's use a hash
+    # of the OAuth secret key to make it unique to the Space + updated in case OAuth
+    # config gets updated.
+    session_secret = OAUTH_CLIENT_SECRET + "-competitions-v1"
+    # ^ if we change the session cookie format in the future, we can bump the version of the session secret to make
+    #   sure cookies are invalidated. Otherwise some users with an old cookie format might get a HTTP 500 error.
+    app.add_middleware(
+        SessionMiddleware,
+        secret_key=hashlib.sha256(session_secret.encode()).hexdigest(),
+        https_only=True,
+        same_site="none",
+    )
+
+
+def _add_oauth_routes(app: fastapi.FastAPI) -> None:
+    """Add OAuth routes to the FastAPI app (login, callback handler and logout)."""
+    # Check environment variables
+    msg = (
+        "OAuth is required but {} environment variable is not set. Make sure you've enabled OAuth in your Space by"
+        " setting `hf_oauth: true` in the Space metadata."
+    )
+    if OAUTH_CLIENT_ID is None:
+        raise ValueError(msg.format("OAUTH_CLIENT_ID"))
+    if OAUTH_CLIENT_SECRET is None:
+        raise ValueError(msg.format("OAUTH_CLIENT_SECRET"))
+    if OAUTH_SCOPES is None:
+        raise ValueError(msg.format("OAUTH_SCOPES"))
+    if OPENID_PROVIDER_URL is None:
+        raise ValueError(msg.format("OPENID_PROVIDER_URL"))
+
+    # Register OAuth server
+    oauth = OAuth()
+    oauth.register(
+        name="huggingface",
+        client_id=OAUTH_CLIENT_ID,
+        client_secret=OAUTH_CLIENT_SECRET,
+        client_kwargs={"scope": OAUTH_SCOPES},
+        server_metadata_url=OPENID_PROVIDER_URL + "/.well-known/openid-configuration",
+    )
+
+    # Define OAuth routes
+    @app.get("/login/huggingface")
+    async def oauth_login(request: fastapi.Request):
+        """Endpoint that redirects to HF OAuth page."""
+        redirect_uri = request.url_for("auth")
+        redirect_uri_as_str = str(redirect_uri)
+        if redirect_uri.netloc.endswith(".hf.space"):
+            redirect_uri_as_str = redirect_uri_as_str.replace("http://", "https://")
+        return await oauth.huggingface.authorize_redirect(request, redirect_uri_as_str)  # type: ignore
+
+    @app.get("/auth")
+    async def auth(request: fastapi.Request) -> RedirectResponse:
+        """Endpoint that handles the OAuth callback."""
+        try:
+            oauth_info = await oauth.huggingface.authorize_access_token(request)  # type: ignore
+        except MismatchingStateError:
+            # If the state mismatch, it is very likely that the cookie is corrupted.
+            # There is a bug reported in authlib that causes the token to grow indefinitely if the user tries to login
+            # repeatedly. Since cookies cannot get bigger than 4kb, the token will be truncated at some point - hence
+            # losing the state. A workaround is to delete the cookie and redirect the user to the login page again.
+            # See https://github.com/lepture/authlib/issues/622 for more details.
+            login_uri = "/login/huggingface"
+            if "_target_url" in request.query_params:
+                login_uri += "?" + urllib.parse.urlencode(  # Keep same _target_url as before
+                    {"_target_url": request.query_params["_target_url"]}
+                )
+            for key in list(request.session.keys()):
+                # Delete all keys that are related to the OAuth state
+                if key.startswith("_state_huggingface"):
+                    request.session.pop(key)
+            return RedirectResponse(login_uri)
+
+        request.session["oauth_info"] = oauth_info
+        return _redirect_to_target(request)
+
+
+def _generate_redirect_uri(request: fastapi.Request) -> str:
+    if "_target_url" in request.query_params:
+        # if `_target_url` already in query params => respect it
+        target = request.query_params["_target_url"]
+    else:
+        # otherwise => keep query params
+        target = "/?" + urllib.parse.urlencode(request.query_params)
+
+    redirect_uri = request.url_for("oauth_redirect_callback").include_query_params(_target_url=target)
+    redirect_uri_as_str = str(redirect_uri)
+    if redirect_uri.netloc.endswith(".hf.space"):
+        # In Space, FastAPI redirect as http but we want https
+        redirect_uri_as_str = redirect_uri_as_str.replace("http://", "https://")
+    return redirect_uri_as_str
+
+
+def _redirect_to_target(request: fastapi.Request, default_target: str = "/") -> RedirectResponse:
+    target = request.query_params.get("_target_url", default_target)
+    return RedirectResponse(target)

competitions/params.py

@@ -0,0 +1,34 @@
+import os
+from typing import List
+
+from pydantic import BaseModel
+
+
+class EvalParams(BaseModel):
+    competition_id: str
+    competition_type: str
+    metric: str
+    token: str
+    team_id: str
+    submission_id: str
+    submission_id_col: str
+    submission_cols: List[str]
+    submission_rows: int
+    output_path: str
+    submission_repo: str
+    time_limit: int
+    dataset: str
+    submission_filenames: List[str]
+
+    class Config:
+        protected_namespaces = ()
+
+    def save(self, output_dir):
+        """
+        Save parameters to a json file.
+        """
+        os.makedirs(output_dir, exist_ok=True)
+        path = os.path.join(output_dir, "params.json")
+        # save formatted json
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(self.model_dump_json(indent=4))

competitions/runner.py

@@ -0,0 +1,230 @@
+import glob
+import io
+import json
+import os
+import time
+from dataclasses import dataclass
+
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download, snapshot_download
+from loguru import logger
+
+from competitions.enums import SubmissionStatus
+from competitions.info import CompetitionInfo
+from competitions.utils import run_evaluation
+
+
+_DOCKERFILE = """
+FROM huggingface/competitions:latest
+
+CMD uvicorn competitions.api:api --port 7860 --host 0.0.0.0
+"""
+
+# format _DOCKERFILE
+_DOCKERFILE = _DOCKERFILE.replace("\n", " ").replace("  ", "\n").strip()
+
+
+@dataclass
+class JobRunner:
+    competition_id: str
+    token: str
+    output_path: str
+
+    def __post_init__(self):
+        self.competition_info = CompetitionInfo(competition_id=self.competition_id, autotrain_token=self.token)
+        self.competition_id = self.competition_info.competition_id
+        self.competition_type = self.competition_info.competition_type
+        self.metric = self.competition_info.metric
+        self.submission_id_col = self.competition_info.submission_id_col
+        self.submission_cols = self.competition_info.submission_cols
+        self.submission_rows = self.competition_info.submission_rows
+        self.time_limit = self.competition_info.time_limit
+        self.dataset = self.competition_info.dataset
+        self.submission_filenames = self.competition_info.submission_filenames
+
+    def get_pending_subs(self):
+        submission_jsons = snapshot_download(
+            repo_id=self.competition_id,
+            allow_patterns="submission_info/*.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        submission_jsons = glob.glob(os.path.join(submission_jsons, "submission_info/*.json"))
+        pending_submissions = []
+        for _json in submission_jsons:
+            _json = json.load(open(_json, "r", encoding="utf-8"))
+            team_id = _json["id"]
+            for sub in _json["submissions"]:
+                if sub["status"] == SubmissionStatus.PENDING.value:
+                    pending_submissions.append(
+                        {
+                            "team_id": team_id,
+                            "submission_id": sub["submission_id"],
+                            "datetime": sub["datetime"],
+                            "submission_repo": sub["submission_repo"],
+                            "space_id": sub["space_id"],
+                        }
+                    )
+        if len(pending_submissions) == 0:
+            return None
+        logger.info(f"Found {len(pending_submissions)} pending submissions.")
+        pending_submissions = pd.DataFrame(pending_submissions)
+        pending_submissions["datetime"] = pd.to_datetime(pending_submissions["datetime"])
+        pending_submissions = pending_submissions.sort_values("datetime")
+        pending_submissions = pending_submissions.reset_index(drop=True)
+        return pending_submissions
+
+    def _queue_submission(self, team_id, submission_id):
+        team_fname = hf_hub_download(
+            repo_id=self.competition_id,
+            filename=f"submission_info/{team_id}.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
+
+        for submission in team_submission_info["submissions"]:
+            if submission["submission_id"] == submission_id:
+                submission["status"] = SubmissionStatus.QUEUED.value
+                break
+
+        team_submission_info_json = json.dumps(team_submission_info, indent=4)
+        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
+        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
+        api = HfApi(token=self.token)
+        api.upload_file(
+            path_or_fileobj=team_submission_info_json_buffer,
+            path_in_repo=f"submission_info/{team_id}.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+
+    def mark_submission_failed(self, team_id, submission_id):
+        team_fname = hf_hub_download(
+            repo_id=self.competition_id,
+            filename=f"submission_info/{team_id}.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
+
+        for submission in team_submission_info["submissions"]:
+            if submission["submission_id"] == submission_id:
+                submission["status"] = SubmissionStatus.FAILED.value
+
+        team_submission_info_json = json.dumps(team_submission_info, indent=4)
+        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
+        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
+
+        api = HfApi(token=self.token)
+        api.upload_file(
+            path_or_fileobj=team_submission_info_json_buffer,
+            path_in_repo=f"submission_info/{team_id}.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+
+    def run_local(self, team_id, submission_id, submission_repo):
+        self._queue_submission(team_id, submission_id)
+        eval_params = {
+            "competition_id": self.competition_id,
+            "competition_type": self.competition_type,
+            "metric": self.metric,
+            "token": self.token,
+            "team_id": team_id,
+            "submission_id": submission_id,
+            "submission_id_col": self.submission_id_col,
+            "submission_cols": self.submission_cols,
+            "submission_rows": self.submission_rows,
+            "output_path": self.output_path,
+            "submission_repo": submission_repo,
+            "time_limit": self.time_limit,
+            "dataset": self.dataset,
+            "submission_filenames": self.submission_filenames,
+        }
+        eval_params = json.dumps(eval_params)
+        eval_pid = run_evaluation(eval_params, local=True, wait=True)
+        logger.info(f"New evaluation process started with pid {eval_pid}.")
+
+    def _create_readme(self, project_name):
+        _readme = "---\n"
+        _readme += f"title: {project_name}\n"
+        _readme += "emoji: 🚀\n"
+        _readme += "colorFrom: green\n"
+        _readme += "colorTo: indigo\n"
+        _readme += "sdk: docker\n"
+        _readme += "pinned: false\n"
+        _readme += "duplicated_from: autotrain-projects/autotrain-advanced\n"
+        _readme += "---\n"
+        _readme = io.BytesIO(_readme.encode())
+        return _readme
+
+    def create_space(self, team_id, submission_id, submission_repo, space_id):
+        api = HfApi(token=self.token)
+        params = {
+            "competition_id": self.competition_id,
+            "competition_type": self.competition_type,
+            "metric": self.metric,
+            "token": self.token,
+            "team_id": team_id,
+            "submission_id": submission_id,
+            "submission_id_col": self.submission_id_col,
+            "submission_cols": self.submission_cols,
+            "submission_rows": self.submission_rows,
+            "output_path": self.output_path,
+            "submission_repo": submission_repo,
+            "time_limit": self.time_limit,
+            "dataset": self.dataset,
+            "submission_filenames": self.submission_filenames,
+        }
+
+        api.add_space_secret(repo_id=space_id, key="PARAMS", value=json.dumps(params))
+
+        readme = self._create_readme(space_id.split("/")[-1])
+        api.upload_file(
+            path_or_fileobj=readme,
+            path_in_repo="README.md",
+            repo_id=space_id,
+            repo_type="space",
+        )
+
+        _dockerfile = io.BytesIO(_DOCKERFILE.encode())
+        api.upload_file(
+            path_or_fileobj=_dockerfile,
+            path_in_repo="Dockerfile",
+            repo_id=space_id,
+            repo_type="space",
+        )
+        self._queue_submission(team_id, submission_id)
+
+    def run(self):
+        while True:
+            pending_submissions = self.get_pending_subs()
+            if pending_submissions is None:
+                time.sleep(5)
+                continue
+            if self.competition_type == "generic":
+                for _, row in pending_submissions.iterrows():
+                    team_id = row["team_id"]
+                    submission_id = row["submission_id"]
+                    submission_repo = row["submission_repo"]
+                    self.run_local(team_id, submission_id, submission_repo)
+            elif self.competition_type == "script":
+                for _, row in pending_submissions.iterrows():
+                    team_id = row["team_id"]
+                    submission_id = row["submission_id"]
+                    submission_repo = row["submission_repo"]
+                    space_id = row["space_id"]
+                    try:
+                        self.create_space(team_id, submission_id, submission_repo, space_id)
+                    except Exception as e:
+                        logger.error(
+                            f"Failed to create space for {team_id} {submission_id} {submission_repo} {space_id}: {e}"
+                        )
+                        # mark submission as failed
+                        self.mark_submission_failed(team_id, submission_id)
+                        logger.error(f"Marked submission {submission_id} as failed.")
+                        continue
+            time.sleep(5)

competitions/submissions.py

@@ -0,0 +1,336 @@
+import io
+import json
+import uuid
+from dataclasses import dataclass
+from datetime import datetime
+
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download
+
+from competitions.enums import SubmissionStatus
+from competitions.errors import AuthenticationError, PastDeadlineError, SubmissionError, SubmissionLimitError
+from competitions.utils import token_information
+
+
+@dataclass
+class Submissions:
+    competition_id: str
+    competition_type: str
+    submission_limit: str
+    hardware: str
+    end_date: datetime
+    token: str
+
+    def _verify_submission(self, bytes_data):
+        return True
+
+    def _num_subs_today(self, todays_date, team_submission_info):
+        todays_submissions = 0
+        for sub in team_submission_info["submissions"]:
+            submission_datetime = sub["datetime"]
+            submission_date = submission_datetime.split(" ")[0]
+            if submission_date == todays_date:
+                todays_submissions += 1
+        return todays_submissions
+
+    def _is_submission_allowed(self, team_id):
+        todays_date = datetime.now()
+        if todays_date > self.end_date:
+            raise PastDeadlineError("Competition has ended.")
+
+        todays_date = todays_date.strftime("%Y-%m-%d")
+        team_submission_info = self._download_team_submissions(team_id)
+
+        if len(team_submission_info["submissions"]) == 0:
+            team_submission_info["submissions"] = []
+
+        todays_submissions = self._num_subs_today(todays_date, team_submission_info)
+        if todays_submissions >= self.submission_limit:
+            return False
+        return True
+
+    def _increment_submissions(
+        self,
+        team_id,
+        user_id,
+        submission_id,
+        submission_comment,
+        submission_repo=None,
+        space_id=None,
+    ):
+        if submission_repo is None:
+            submission_repo = ""
+        if space_id is None:
+            space_id = ""
+        team_fname = hf_hub_download(
+            repo_id=self.competition_id,
+            filename=f"submission_info/{team_id}.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
+        datetime_now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        # here goes all the default stuff for submission
+        team_submission_info["submissions"].append(
+            {
+                "datetime": datetime_now,
+                "submission_id": submission_id,
+                "submission_comment": submission_comment,
+                "submission_repo": submission_repo,
+                "space_id": space_id,
+                "submitted_by": user_id,
+                "status": SubmissionStatus.PENDING.value,
+                "selected": False,
+                "public_score": {},
+                "private_score": {},
+            }
+        )
+        # count the number of times user has submitted today
+        todays_date = datetime.now().strftime("%Y-%m-%d")
+        todays_submissions = self._num_subs_today(todays_date, team_submission_info)
+        self._upload_team_submissions(team_id, team_submission_info)
+        return todays_submissions
+
+    def _upload_team_submissions(self, team_id, team_submission_info):
+        team_submission_info_json = json.dumps(team_submission_info, indent=4)
+        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
+        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
+        api = HfApi(token=self.token)
+        api.upload_file(
+            path_or_fileobj=team_submission_info_json_buffer,
+            path_in_repo=f"submission_info/{team_id}.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+
+    def _download_team_submissions(self, team_id):
+        team_fname = hf_hub_download(
+            repo_id=self.competition_id,
+            filename=f"submission_info/{team_id}.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
+        return team_submission_info
+
+    def update_selected_submissions(self, user_token, selected_submission_ids):
+        current_datetime = datetime.now()
+        if current_datetime > self.end_date:
+            raise PastDeadlineError("Competition has ended.")
+
+        user_info = self._get_user_info(user_token)
+        team_id = self._get_team_id(user_info, create_team=False)
+        team_submission_info = self._download_team_submissions(team_id)
+
+        for sub in team_submission_info["submissions"]:
+            if sub["submission_id"] in selected_submission_ids:
+                sub["selected"] = True
+            else:
+                sub["selected"] = False
+
+        self._upload_team_submissions(team_id, team_submission_info)
+
+    def _get_team_subs(self, team_id, private=False):
+        team_submissions_info = self._download_team_submissions(team_id)
+        submissions_df = pd.DataFrame(team_submissions_info["submissions"])
+
+        if len(submissions_df) == 0:
+            return pd.DataFrame(), pd.DataFrame()
+
+        if not private:
+            submissions_df = submissions_df.drop(columns=["private_score"])
+
+        submissions_df = submissions_df.sort_values(by="datetime", ascending=False)
+        submissions_df = submissions_df.reset_index(drop=True)
+
+        # stringify public_score column
+        submissions_df["public_score"] = submissions_df["public_score"].apply(json.dumps)
+
+        if private:
+            submissions_df["private_score"] = submissions_df["private_score"].apply(json.dumps)
+
+        submissions_df["status"] = submissions_df["status"].apply(lambda x: SubmissionStatus(x).name)
+
+        return submissions_df
+
+    def _get_user_info(self, user_token):
+        user_info = token_information(token=user_token)
+        if "error" in user_info:
+            raise AuthenticationError("Invalid token")
+
+        # if user_info["emailVerified"] is False:
+        #     raise AuthenticationError("Please verify your email on Hugging Face Hub")
+        return user_info
+
+    def my_submissions(self, user_token):
+        user_info = self._get_user_info(user_token)
+        current_date_time = datetime.now()
+        private = False
+        if current_date_time >= self.end_date:
+            private = True
+        team_id = self._get_team_id(user_info, create_team=False)
+        if not team_id:
+            return pd.DataFrame()
+        return self._get_team_subs(team_id, private=private)
+
+    def _create_team(self, user_team, user_id, user_name):
+        team_metadata = hf_hub_download(
+            repo_id=self.competition_id,
+            filename="teams.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+
+        with open(team_metadata, "r", encoding="utf-8") as f:
+            team_metadata = json.load(f)
+
+        # create a new team, if user is not in any team
+        team_id = str(uuid.uuid4())
+        user_team[user_id] = team_id
+
+        team_metadata[team_id] = {
+            "id": team_id,
+            "name": user_name,
+            "members": [user_id],
+            "leader": user_id,
+        }
+
+        user_team_json = json.dumps(user_team, indent=4)
+        user_team_json_bytes = user_team_json.encode("utf-8")
+        user_team_json_buffer = io.BytesIO(user_team_json_bytes)
+
+        team_metadata_json = json.dumps(team_metadata, indent=4)
+        team_metadata_json_bytes = team_metadata_json.encode("utf-8")
+        team_metadata_json_buffer = io.BytesIO(team_metadata_json_bytes)
+
+        team_submission_info = {}
+        team_submission_info["id"] = team_id
+        team_submission_info["submissions"] = []
+        team_submission_info_json = json.dumps(team_submission_info, indent=4)
+        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
+        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
+
+        api = HfApi(token=self.token)
+        api.upload_file(
+            path_or_fileobj=user_team_json_buffer,
+            path_in_repo="user_team.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+        api.upload_file(
+            path_or_fileobj=team_metadata_json_buffer,
+            path_in_repo="teams.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+        api.upload_file(
+            path_or_fileobj=team_submission_info_json_buffer,
+            path_in_repo=f"submission_info/{team_id}.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+        return team_id
+
+    def _get_team_id(self, user_info, create_team):
+        user_id = user_info["id"]
+        user_name = user_info["name"]
+        user_team = hf_hub_download(
+            repo_id=self.competition_id,
+            filename="user_team.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        with open(user_team, "r", encoding="utf-8") as f:
+            user_team = json.load(f)
+
+        if user_id in user_team:
+            return user_team[user_id]
+
+        if create_team is False:
+            return None
+
+        # if user_id is not there in user_team, create a new team
+        team_id = self._create_team(user_team, user_id, user_name)
+        return team_id
+
+    def new_submission(self, user_token, uploaded_file, submission_comment):
+        # verify token
+        user_info = self._get_user_info(user_token)
+        submission_id = str(uuid.uuid4())
+        user_id = user_info["id"]
+        team_id = self._get_team_id(user_info, create_team=True)
+
+        # check if team can submit to the competition
+        if self._is_submission_allowed(team_id) is False:
+            raise SubmissionLimitError("Submission limit reached")
+
+        if self.competition_type == "generic":
+            bytes_data = uploaded_file.file.read()
+            # verify file is valid
+            if not self._verify_submission(bytes_data):
+                raise SubmissionError("Invalid submission file")
+
+            file_extension = uploaded_file.filename.split(".")[-1]
+            # upload file to hf hub
+            api = HfApi(token=self.token)
+            api.upload_file(
+                path_or_fileobj=bytes_data,
+                path_in_repo=f"submissions/{team_id}-{submission_id}.{file_extension}",
+                repo_id=self.competition_id,
+                repo_type="dataset",
+            )
+            submissions_made = self._increment_submissions(
+                team_id=team_id,
+                user_id=user_id,
+                submission_id=submission_id,
+                submission_comment=submission_comment,
+            )
+        else:
+            # Download the submission repo and upload it to the competition repo
+            # submission_repo = snapshot_download(
+            #     repo_id=uploaded_file,
+            #     local_dir=submission_id,
+            #     token=user_token,
+            #     repo_type="model",
+            # )
+            # api = HfApi(token=self.token)
+            # competition_user = self.competition_id.split("/")[0]
+            # api.create_repo(
+            #     repo_id=f"{competition_user}/{submission_id}",
+            #     repo_type="model",
+            #     private=True,
+            # )
+            # api.upload_folder(
+            #     folder_path=submission_repo,
+            #     repo_id=f"{competition_user}/{submission_id}",
+            #     repo_type="model",
+            # )
+            # create barebones submission runner space
+            user_api = HfApi(token=user_token)
+            # submission_id is the sha of the submitted model repo + "__" + submission_id
+            submission_id = user_api.model_info(repo_id=uploaded_file).sha + "__" + submission_id
+            competition_organizer = self.competition_id.split("/")[0]
+            space_id = f"{competition_organizer}/comp-{submission_id}"
+            api = HfApi(token=self.token)
+            api.create_repo(
+                repo_id=space_id,
+                repo_type="space",
+                space_sdk="docker",
+                space_hardware=self.hardware,
+                private=True,
+            )
+
+            api.add_space_secret(repo_id=space_id, key="USER_TOKEN", value=user_token)
+            submissions_made = self._increment_submissions(
+                team_id=team_id,
+                user_id=user_id,
+                submission_id=submission_id,
+                submission_comment=submission_comment,
+                submission_repo=uploaded_file,
+                space_id=space_id,
+            )
+        remaining_submissions = self.submission_limit - submissions_made
+        return remaining_submissions

competitions/templates/index.html

@@ -0,0 +1,951 @@
+<!doctype html>
+<html>
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
+    <script src="https://cdn.tailwindcss.com?plugins=forms,typography,aspect-ratio,line-clamp"></script>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.min.css">
+    <link href="https://cdnjs.cloudflare.com/ajax/libs/flowbite/2.2.1/flowbite.min.css" rel="stylesheet" />
+    <script>
+        document.addEventListener('DOMContentLoaded', function () {
+            function showSubmissionModal() {
+                const modal = document.getElementById('submission-modal');
+                modal.classList.add('flex');
+                modal.classList.remove('hidden');
+            }
+
+            function hideSubmissionModal() {
+                const modal = document.getElementById('submission-modal');
+                modal.classList.remove('flex');
+                modal.classList.add('hidden');
+            }
+
+            function addTargetBlankToLinks() {
+                const content = document.getElementById('content');
+                const links = content.getElementsByTagName('a');
+
+                for (let i = 0; i < links.length; i++) {
+                    if (!links[i].hasAttribute('target')) {
+                        links[i].setAttribute('target', '_blank');
+                    }
+                }
+            }
+
+            function fetchAndDisplayCompetitionInfo() {
+                const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+                articleLoadingSpinner.classList.remove('hidden');
+                fetch('/competition_info')
+                    .then(response => {
+                        if (!response.ok) {
+                            throw new Error('Network response was not ok');
+                        }
+                        return response.json(); // Parse the JSON response
+                    })
+                    .then(data => {
+                        // Populate the 'content' div with the HTML from the response
+                        const contentDiv = document.getElementById('content');
+                        contentDiv.style.display = 'block';
+                        contentDiv.innerHTML = marked.parse(data.response);
+                        addTargetBlankToLinks();
+                        articleLoadingSpinner.classList.add('hidden');
+                    })
+                    .catch(error => {
+                        console.error('There has been a problem with your fetch operation:', error);
+                        articleLoadingSpinner.classList.add('hidden');
+                    });
+            }
+
+            function fetchAndDisplayDatasetInfo() {
+                const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+                articleLoadingSpinner.classList.remove('hidden');
+                fetch('/dataset_info')
+                    .then(response => {
+                        if (!response.ok) {
+                            throw new Error('Network response was not ok');
+                        }
+                        return response.json(); // Parse the JSON response
+                    })
+                    .then(data => {
+                        // Populate the 'content' div with the HTML from the response
+                        const contentDiv = document.getElementById('content');
+                        contentDiv.innerHTML = marked.parse(data.response);
+                        addTargetBlankToLinks();
+                        articleLoadingSpinner.classList.add('hidden');
+                    })
+                    .catch(error => {
+                        console.error('There has been a problem with your fetch operation:', error);
+                        articleLoadingSpinner.classList.add('hidden');
+                    });
+            }
+            function fetchAndDisplayLeaderboard(leaderboardType) {
+                const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+                articleLoadingSpinner.classList.remove('hidden');
+
+                const payload = {
+                    lb: leaderboardType,
+                };
+
+                fetch('/leaderboard', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify(payload)
+                })
+                    .then(response => {
+                        if (!response.ok) {
+                            throw new Error('Network response was not ok');
+                        }
+                        return response.json();
+                    })
+                    .then(data => {
+                        const contentDiv = document.getElementById('content');
+                        contentDiv.innerHTML = marked.parse(data.response);
+                        articleLoadingSpinner.classList.add('hidden');
+                    })
+                    .catch(error => {
+                        console.error('There has been a problem with your fetch operation:', error);
+                        articleLoadingSpinner.classList.add('hidden');
+                    });
+            }
+
+            function fetchAndDisplayPublicLeaderboard() {
+                fetchAndDisplayLeaderboard('public');
+            }
+
+            function fetchAndDisplayPrivateLeaderboard() {
+                fetchAndDisplayLeaderboard('private');
+            }
+
+            function fetchAndDisplaySubmissions() {
+                const apiEndpoint = '/my_submissions';
+                const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+                articleLoadingSpinner.classList.remove('hidden');
+
+                const requestOptions = {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    }
+                };
+
+                fetch(apiEndpoint, requestOptions)
+                    .then(response => {
+                        if (!response.ok) {
+                            throw new Error('Network response was not ok');
+                        }
+                        return response.json();
+                    })
+                    .then(data => {
+                        const contentDiv = document.getElementById('content');
+                        // console.log(data.response.submissions);
+                        // contentDiv.innerHTML = marked.parse(data.response.submission_text) + data.response.submissions;
+                        if (data.response.submissions && data.response.submissions.length > 0 && data.response.error.length == 0) {
+                            // Start building the table HTML
+                            let tableHTML = `
+                                <div class="flex items-center">
+                                    <input type="text" name="team_name" id="team_name" class="mt-1 mb-1 block me-2" value="${data.response.team_name}">
+                                    <button id="updateTeamNameButton" type="button" class="confirm text-white bg-green-600 hover:bg-green-800 focus:ring-4 focus:outline-none focus:ring-green-300 font-medium rounded-lg text-sm inline-flex items-center px-5 py-2.5 text-center me-2">Update Team Name</button>
+                                </div>`;
+                            tableHTML += '<table border="1"><tr><th>Datetime</th><th>Submission ID</th><th>Public Score</th><th>Submission Comment</th><th>Selected</th><th>Status</th></tr>';
+
+                            // Iterate over each submission and add it to the table
+                            data.response.submissions.forEach(submission => {
+                                tableHTML += `<tr>
+                                    <td>${submission.datetime}</td>
+                                    <td>${submission.submission_id}</td>
+                                    <td>${submission.public_score}</td>
+                                    <td>${submission.submission_comment}</td>
+                                    <td><input type="checkbox" name="selectedSubmissions" value="${submission.submission_id}" ${submission.selected ? 'checked' : ''}></td>
+                                    <td>${submission.status}</td>
+                                  </tr>`;
+                            });
+
+                            // Close the table HTML and set it as the content
+                            tableHTML += '</table>';
+                            tableHTML += '<button id="updateSelectedSubmissionsButton" type="button" class="confirm text-white bg-green-600 hover:bg-green-800 focus:ring-4 focus:outline-none focus:ring-green-300 font-medium rounded-lg text-sm inline-flex items-center px-5 py-2.5 text-center me-2">Update Selected Submissions</button>';
+                            // add a text field which displays team name and a button to update team name
+                            contentDiv.innerHTML = marked.parse(data.response.submission_text) + tableHTML;
+                            document.getElementById('updateSelectedSubmissionsButton').addEventListener('click', function () {
+                                updateSelectedSubmissions();
+                            });
+                            document.getElementById('updateTeamNameButton').addEventListener('click', function () {
+                                updateTeamName();
+                            });
+                        } else {
+                            // Display message if there are no submissions
+                            contentDiv.innerHTML = marked.parse(data.response.submission_text) + marked.parse(data.response.error);
+                        }
+                        articleLoadingSpinner.classList.add('hidden');
+                    })
+                    .catch(error => {
+                        console.error('There was a problem with the fetch operation:', error);
+                        articleLoadingSpinner.classList.add('hidden');
+                    });
+            }
+
+            function fetchAndDisplaySubmissionInfo() {
+                const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+                articleLoadingSpinner.classList.remove('hidden');
+                fetch('/submission_info')
+                    .then(response => {
+                        if (!response.ok) {
+                            throw new Error('Network response was not ok');
+                        }
+                        return response.json(); // Parse the JSON response
+                    })
+                    .then(data => {
+                        // Populate the 'content' div with the HTML from the response
+                        const contentDiv = document.getElementById('content');
+                        contentDiv.innerHTML = marked.parse(data.response);
+                        addTargetBlankToLinks();
+                        articleLoadingSpinner.classList.add('hidden');
+                    })
+                    .catch(error => {
+                        console.error('There has been a problem with your fetch operation:', error);
+                        articleLoadingSpinner.classList.add('hidden');
+                    });
+            }
+
+            function fetchAndDisplayTeamInfo() {
+                const apiEndpoint = '/team_info';
+                const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+                articleLoadingSpinner.classList.remove('hidden');
+
+                const requestOptions = {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    }
+                };
+                fetch(apiEndpoint, requestOptions)
+                    .then(response => {
+                        if (!response.ok) {
+                            throw new Error('Network response was not ok');
+                        }
+                        return response.json(); // Parse the JSON response
+                    })
+                    .then(data => {
+                        // Populate the 'content' div with the HTML from the response
+                        const contentDiv = document.getElementById('content');
+                        if (data.team_exists) {
+                            contentHTML = "<h2>Team</h2>";
+                            contentHTML += "<p>" + data.team_name + "</p>";
+                            contentDiv.innerHTML = marked.parse(contentHTML);
+                        } else {
+                            contentDiv.innerHTML = marked.parse(data.response);
+                        }
+                        contentDiv.innerHTML = marked.parse(data.response);
+                        articleLoadingSpinner.classList.add('hidden');
+                    })
+                    .catch(error => {
+                        console.error('There has been a problem with your fetch operation:', error);
+                        articleLoadingSpinner.classList.add('hidden');
+                    });
+            }
+
+            function fetchAndDisplayRules() {
+                const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+                articleLoadingSpinner.classList.remove('hidden');
+                fetch('/rules')
+                    .then(response => {
+                        if (!response.ok) {
+                            throw new Error('Network response was not ok');
+                        }
+                        return response.json(); // Parse the JSON response
+                    })
+                    .then(data => {
+                        // Populate the 'content' div with the HTML from the response
+                        const contentDiv = document.getElementById('content');
+                        contentDiv.innerHTML = marked.parse(data.response);
+                        addTargetBlankToLinks();
+                        articleLoadingSpinner.classList.add('hidden');
+                    })
+                    .catch(error => {
+                        console.error('There has been a problem with your fetch operation:', error);
+                        articleLoadingSpinner.classList.add('hidden');
+                    });
+            }
+
+            const homeLink = document.getElementById('home');
+            const datasetLink = document.getElementById('dataset');
+            const publicLBLink = document.getElementById('public_lb');
+            const privateLBLink = document.getElementById('private_lb');
+            const newSubmission = document.getElementById('new_submission');
+            const mySubmissions = document.getElementById('my_submissions');
+            const submissionInfo = document.getElementById('submission_info');
+            const rulesLink = document.getElementById('rules');
+
+            // Add a click event listener to the 'Home' link
+            homeLink.addEventListener('click', function (event) {
+                event.preventDefault(); // Prevent the default link behavior
+                fetchAndDisplayCompetitionInfo(); // Fetch and display info on click
+            });
+
+            datasetLink.addEventListener('click', function (event) {
+                event.preventDefault(); // Prevent the default link behavior
+                fetchAndDisplayDatasetInfo(); // Fetch and display info on click
+            });
+
+            publicLBLink.addEventListener('click', function (event) {
+                event.preventDefault(); // Prevent the default link behavior
+                fetchAndDisplayPublicLeaderboard(); // Fetch and display info on click
+            });
+
+            privateLBLink.addEventListener('click', function (event) {
+                event.preventDefault(); // Prevent the default link behavior
+                fetchAndDisplayPrivateLeaderboard(); // Fetch and display info on click
+            });
+
+            newSubmission.addEventListener('click', function (event) {
+                event.preventDefault(); // Prevent the default link behavior
+                showSubmissionModal(); // Fetch and display info on click
+            });
+            mySubmissions.addEventListener('click', function (event) {
+                event.preventDefault(); // Prevent the default link behavior
+                fetchAndDisplaySubmissions(); // Fetch and display info on click
+            });
+            submissionInfo.addEventListener('click', function (event) {
+                event.preventDefault(); // Prevent the default link behavior
+                fetchAndDisplaySubmissionInfo(); // Fetch and display info on click
+            });
+            rulesLink.addEventListener('click', function (event) {
+                event.preventDefault(); // Prevent the default link behavior
+                fetchAndDisplayRules(); // Fetch and display info on click
+            });
+
+
+            // Fetch and display info when the page loads
+            fetchAndDisplayCompetitionInfo();
+
+            document.querySelector('#submission-modal .cancel').addEventListener('click', function () {
+                hideSubmissionModal();
+            });
+        });
+
+    </script>
+
+    <script>
+        function makeApiRequest(url, callback) {
+            var xhr = new XMLHttpRequest();
+            xhr.open("GET", url, true);
+            xhr.onreadystatechange = function () {
+                if (xhr.readyState === 4 && xhr.status === 200) {
+                    var response = JSON.parse(xhr.responseText);
+                    callback(response.response);
+                }
+            };
+            xhr.send();
+        }
+
+        function checkOAuth() {
+            var url = "/login_status";
+            makeApiRequest(url, function (response) {
+                if (response === 1) {
+                    document.getElementById("loginButton").style.display = "block";
+                    document.getElementById("logoutButton").style.display = "none";
+                } else if (response === 2) {
+                    document.getElementById("loginButton").style.display = "none";
+                    document.getElementById("logoutButton").style.display = "block";
+                }
+            });
+        }
+        window.onload = checkOAuth;
+    </script>
+</head>
+
+<body class="flex h-screen">
+    <!-- Sidebar -->
+    <aside id="sidebar-multi-level-sidebar"
+        class="fixed top-0 left-0 z-40 w-64 h-screen transition-transform -translate-x-full sm:translate-x-0"
+        aria-label="Sidebar">
+        <div class="h-full px-3 py-4 overflow-y-auto">
+            <ul class="space-y-2 font-medium">
+                <li>
+                    <a href="#" id="home"
+                        class="flex items-center p-2 text-gray-900 rounded-lg hover:bg-gray-100 group">
+                        <svg class="w-5 h-5 text-gray-500 transition duration-75 group-hover:text-gray-900"
+                            viewBox="0 0 22 21" xmlns="http://www.w3.org/2000/svg" fill="currentColor">
+                            <path d="M1,10 L11,1 L21,10 L21,20 L1,20 Z" /> <!-- House structure -->
+                            <path d="M6,20 L6,14 L16,14 L16,20" /> <!-- Door -->
+                        </svg>
+
+                        <span class="ms-3">Home</span>
+                    </a>
+                </li>
+                <li>
+                    <a href="#" id="dataset"
+                        class="flex items-center p-2 text-gray-900 rounded-lg hover:bg-gray-100 group">
+                        <svg class="flex-shrink-0 w-5 h-5 text-gray-500 transition duration-75 group-hover:text-gray-900"
+                            aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="currentColor"
+                            viewBox="0 0 18 18">
+                            <path
+                                d="M6.143 0H1.857A1.857 1.857 0 0 0 0 1.857v4.286C0 7.169.831 8 1.857 8h4.286A1.857 1.857 0 0 0 8 6.143V1.857A1.857 1.857 0 0 0 6.143 0Zm10 0h-4.286A1.857 1.857 0 0 0 10 1.857v4.286C10 7.169 10.831 8 11.857 8h4.286A1.857 1.857 0 0 0 18 6.143V1.857A1.857 1.857 0 0 0 16.143 0Zm-10 10H1.857A1.857 1.857 0 0 0 0 11.857v4.286C0 17.169.831 18 1.857 18h4.286A1.857 1.857 0 0 0 8 16.143v-4.286A1.857 1.857 0 0 0 6.143 10Zm10 0h-4.286A1.857 1.857 0 0 0 10 11.857v4.286c0 1.026.831 1.857 1.857 1.857h4.286A1.857 1.857 0 0 0 18 16.143v-4.286A1.857 1.857 0 0 0 16.143 10Z" />
+                        </svg>
+                        <span class="flex-1 ms-3 whitespace-nowrap">Dataset</span>
+                    </a>
+                </li>
+                {% if rules_available %}
+                <li>
+                    <a href="#" id="rules"
+                        class="flex items-center p-2 text-gray-900 rounded-lg hover:bg-gray-100 group">
+                        <svg class="flex-shrink-0 w-5 h-5 text-gray-500 transition duration-75 group-hover:text-gray-900"
+                            aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="currentColor"
+                            viewBox="0 0 24 24">
+                            <path
+                                d="M3 0h18v2H3V0zm0 4h18v2H3V4zm0 4h18v2H3V8zm0 4h18v2H3v-2zm0 4h18v2H3v-2zm0 4h18v2H3v-2z" />
+                        </svg>
+                        <span class="flex-1 ms-3 whitespace-nowrap">Rules</span>
+                    </a>
+                </li>
+                {% else %}
+                <span id="rules"></span>
+                {% endif %}
+                <li>
+                    <button type="button"
+                        class="flex items-center w-full p-2 text-base text-gray-900 transition duration-75 rounded-lg group hover:bg-gray-100"
+                        aria-controls="lb-dropdown" data-collapse-toggle="lb-dropdown">
+                        <svg class="flex-shrink-0 w-5 h-5 text-gray-500 transition duration-75 group-hover:text-gray-900"
+                            aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="currentColor"
+                            viewBox="0 0 18 21">
+                            <path d="M2,4 L20,4 L20,16 L2,16 Z" />
+                            <path d="M6,17 L16,17 L16,18 L6,18 Z" />
+                        </svg>
+                        <span class="flex-1 ms-3 text-left rtl:text-right whitespace-nowrap">Leaderboard</span>
+                        <svg class="w-3 h-3" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none"
+                            viewBox="0 0 10 6">
+                            <path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                                d="m1 1 4 4 4-4" />
+                        </svg>
+                    </button>
+                    <ul id="lb-dropdown" class="py-2 space-y-2">
+                        <li>
+                            <a href="#" id="public_lb"
+                                class="flex items-center w-full p-2 text-gray-900 transition duration-75 rounded-lg pl-11 group hover:bg-gray-100">Public</a>
+                        </li>
+                        <li>
+                            <a href="#" id="private_lb"
+                                class="flex items-center w-full p-2 text-gray-900 transition duration-75 rounded-lg pl-11 group hover:bg-gray-100">Private</a>
+                        </li>
+                    </ul>
+                </li>
+                <li>
+                    <button type="button"
+                        class="flex items-center w-full p-2 text-base text-gray-900 transition duration-75 rounded-lg group hover:bg-gray-100"
+                        aria-controls="submissions-dropdown" data-collapse-toggle="submissions-dropdown">
+                        <svg class="flex-shrink-0 w-5 h-5 text-gray-500 transition duration-75 group-hover:text-gray-900"
+                            aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="currentColor"
+                            viewBox="0 0 20 20">
+                            <path d="M5 5V.13a2.96 2.96 0 0 0-1.293.749L.879 3.707A2.96 2.96 0 0 0 .13 5H5Z" />
+                            <path
+                                d="M6.737 11.061a2.961 2.961 0 0 1 .81-1.515l6.117-6.116A4.839 4.839 0 0 1 16 2.141V2a1.97 1.97 0 0 0-1.933-2H7v5a2 2 0 0 1-2 2H0v11a1.969 1.969 0 0 0 1.933 2h12.134A1.97 1.97 0 0 0 16 18v-3.093l-1.546 1.546c-.413.413-.94.695-1.513.81l-3.4.679a2.947 2.947 0 0 1-1.85-.227 2.96 2.96 0 0 1-1.635-3.257l.681-3.397Z" />
+                            <path
+                                d="M8.961 16a.93.93 0 0 0 .189-.019l3.4-.679a.961.961 0 0 0 .49-.263l6.118-6.117a2.884 2.884 0 0 0-4.079-4.078l-6.117 6.117a.96.96 0 0 0-.263.491l-.679 3.4A.961.961 0 0 0 8.961 16Zm7.477-9.8a.958.958 0 0 1 .68-.281.961.961 0 0 1 .682 1.644l-.315.315-1.36-1.36.313-.318Zm-5.911 5.911 4.236-4.236 1.359 1.359-4.236 4.237-1.7.339.341-1.699Z" />
+                        </svg>
+                        <span class="flex-1 ms-3 text-left rtl:text-right whitespace-nowrap">Submissions</span>
+                        <svg class="w-3 h-3" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none"
+                            viewBox="0 0 10 6">
+                            <path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                                d="m1 1 4 4 4-4" />
+                        </svg>
+                    </button>
+                    <ul id="submissions-dropdown" class="py-2 space-y-2">
+                        <li>
+                            <a href="#" id="submission_info"
+                                class="flex items-center w-full p-2 text-gray-900 transition duration-75 rounded-lg pl-11 group hover:bg-gray-100">Submission
+                                information</a>
+                        </li>
+                        <li>
+                            <a href="#" id="my_submissions"
+                                class="flex items-center w-full p-2 text-gray-900 transition duration-75 rounded-lg pl-11 group hover:bg-gray-100">My
+                                submissions</a>
+                        </li>
+                        <li>
+                            <a href="#" id="new_submission"
+                                class="flex items-center w-full p-2 text-gray-900 transition duration-75 rounded-lg pl-11 group hover:bg-gray-100">New
+                                submission</a>
+                        </li>
+                    </ul>
+                </li>
+                <li>
+                    <a href="#" id="admin"
+                        class="flex items-center p-2 text-gray-900 rounded-lg hover:bg-gray-100 group">
+                        <svg class="flex-shrink-0 w-5 h-5 text-gray-500 transition duration-75 group-hover:text-gray-900"
+                            aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="currentColor"
+                            viewBox="0 0 24 24">
+                            <path
+                                d="M12 15.5c-1.93 0-3.5-1.57-3.5-3.5s1.57-3.5 3.5-3.5 3.5 1.57 3.5 3.5-1.57 3.5-3.5 3.5zm7.43-3.5c.04-.33.07-.66.07-1s-.03-.67-.07-1l2.11-1.65c.19-.15.23-.42.12-.63l-2-3.46c-.11-.21-.35-.3-.57-.24l-2.49 1c-.52-.4-1.08-.73-1.69-.98l-.38-2.65C14.57 2.18 14.3 2 14 2h-4c-.3 0-.57.18-.64.45L8.98 5.1c-.61.25-1.17.58-1.69.98l-2.49-1c-.22-.06-.46.03-.57.24l-2 3.46c-.11.21-.07.48.12.63l2.11 1.65c-.04.33-.07.66-.07 1s.03.67.07 1L2.46 14.1c-.19.15-.23.42-.12.63l2 3.46c.11.21.35.3.57.24l2.49-1c.52.4 1.08.73 1.69.98l.38 2.65c.07.27.34.45.64.45h4c.3 0 .57-.18.64-.45l.38-2.65c.61-.25 1.17-.58 1.69-.98l2.49 1c.22.06.46-.03.57-.24l2-3.46c.11-.21.07-.48-.12-.63l-2.11-1.65zM12 17c-2.76 0-5-2.24-5-5s2.24-5 5-5 5 2.24 5 5-2.24 5-5 5z" />
+                        </svg>
+                        <span class="flex-1 ms-3 whitespace-nowrap">Admin</span>
+                    </a>
+                </li>
+                <li id="loginButton" style="display: none;">
+                    <a href="/login/huggingface"
+                        class="flex justify-center items-center bg-blue-400 hover:bg-blue-600 text-white text-center font-bold py-2 px-4 rounded transition duration-200 ease-in-out">
+                        Login with Hugging Face
+                    </a>
+                </li>
+                <li id="logoutButton" style="display: none;">
+                    <a href="/logout"
+                        class="flex justify-center items-center bg-red-400 hover:bg-red-600 text-white text-center font-bold py-2 px-4 rounded transition duration-200 ease-in-out">
+                        Logout
+                    </a>
+                </li>
+            </ul>
+
+            <footer>
+                <div class="w-full mx-auto max-w-screen-xl p-4 md:flex md:items-center md:justify-between">
+                    <span class="text-sm text-gray-500 sm:text-center">Powered by <a
+                            href="https://github.com/huggingface/competitions" target="_blank"
+                            class="hover:underline">Hugging Face
+                            Competitions</a>
+                    </span>
+                </div>
+                <div class="text-center">
+                    <span class="text-xs text-gray-400">{{version}}
+                    </span>
+                </div>
+            </footer>
+        </div>
+    </aside>
+    <div class="p-1 sm:ml-64">
+        <img src={{logo}} alt="Competition logo">
+        <hr class="mt-3 mb-2">
+        <div id="articleLoadingSpinner" role="status"
+            class="hidden absolute -translate-x-1/2 -translate-y-1/2 top-2/4 left-1/2">
+            <div class="animate-spin rounded-full h-32 w-32 border-b-2 border-gray-900"></div>
+            <span class="sr-only">Loading...</span>
+        </div>
+        <article class="prose w-full mx-auto max-w-screen-xl p-4 md:flex md:items-center md:justify-between"
+            id="content">
+        </article>
+    </div>
+    <div id="submission-modal" tabindex="-1"
+        class="hidden overflow-y-auto overflow-x-hidden fixed top-0 right-0 left-0 z-50 justify-center items-center w-full md:inset-0 h-[calc(100%-1rem)] max-h-full">
+        <div id="loadingSpinner" role="status"
+            class="hidden absolute -translate-x-1/2 -translate-y-1/2 top-2/4 left-1/2">
+            <div class="animate-spin rounded-full h-32 w-32 border-b-2 border-gray-900"></div>
+            <span class="sr-only">Loading...</span>
+        </div>
+        <div class="form-container max-w-5xl mx-auto mt-3 p-6 shadow-2xl bg-white">
+            <p class="text-lg font-medium text-gray-900">New Submission</p>
+            <form action="#" method="post" class="gap-2" enctype="multipart/form-data">
+                {% if competition_type == 'generic' %}
+                <div class="form-group">
+                    <label class="block mb-2 text-sm font-medium text-gray-900" for="submission_file">Upload
+                        file</label>
+                    <input
+                        class="block w-full text-sm text-gray-900 border border-gray-300 rounded-lg cursor-pointer bg-gray-50 focus:outline-none "
+                        id="submission_file" type="file" name="submission_file">
+                </div>
+                {% endif %}
+                {% if competition_type == 'script' %}
+                <div class="form-group">
+                    <label for="hub_model" class="text-sm font-medium text-gray-700">Hub model
+                    </label>
+                    <input type="text" name="hub_model" id="hub_model"
+                        class="mt-1 block w-full border border-gray-300 px-3 py-1.5 bg-white rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500"
+                        placeholder="username/my-model">
+                </div>
+                {% endif %}
+                <div class="form-group mt-2">
+                    <label for="submission_comment" class="text-sm font-medium text-gray-700">Submission description
+                        (optional)
+                    </label>
+                    <textarea id="submission_comment" name="submission_comment" rows="5"
+                        class="p-2.5 w-full text-sm text-gray-900" placeholder=" "></textarea>
+                </div>
+                <div class="form-actions mt-6">
+                    <button data-modal-hide="submission-modal" type="button"
+                        class="confirm text-white bg-green-600 hover:bg-green-800 focus:ring-4 focus:outline-none focus:ring-green-300font-medium rounded-lg text-sm inline-flex items-center px-5 py-2.5 text-center me-2">
+                        Submit
+                    </button>
+                    <button data-modal-hide="submission-modal" type="button"
+                        class="cancel text-white bg-red-600 hover:bg-red-100 focus:ring-4 focus:outline-none focus:ring-red-200 rounded-lg border border-red-200 text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10">Cancel</button>
+                </div>
+            </form>
+            <hr class="mt-3">
+            <div id="error-message" style="color: red;"></div>
+            <div id="success-message" style="color: green;"></div>
+        </div>
+    </div>
+    <div id="admin-modal" tabindex="-1"
+        class="hidden fixed inset-0 z-40 flex items-center justify-center w-full h-full bg-black bg-opacity-50">
+        <div id="adminLoadingSpinner" role="status"
+            class="hidden fixed inset-0 z-50 flex items-center justify-center bg-black bg-opacity-50">
+            <div class="animate-spin rounded-full h-32 w-32 border-b-2 border-gray-900"></div>
+            <span class="sr-only">Loading...</span>
+        </div>
+        <div class="relative w-full max-w-5xl p-4">
+            <div class="relative bg-white rounded-lg shadow-2xl">
+                <button type="button"
+                    class="absolute top-3 right-3 text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm w-8 h-8 inline-flex justify-center items-center"
+                    data-modal-hide="admin-modal">
+                    <svg class="w-4 h-4" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none"
+                        viewBox="0 0 14 14">
+                        <path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                            d="m1 1 6 6m0 0 6 6M7 7l6-6M7 7l-6 6" />
+                    </svg>
+                    <span class="sr-only">Close</span>
+                </button>
+                <div class="p-6 md:p-8 text-center">
+                    <h3 class="mb-5 text-lg font-medium text-gray-900">Admin</h3>
+                    <div class="tabs">
+                        <ul class="flex border-b">
+                            <li class="mr-1">
+                                <a class="tab bg-white inline-block py-2 px-4 text-blue-500 hover:text-blue-800 font-semibold"
+                                    href="#config">Config</a>
+                            </li>
+                            <li class="mr-1">
+                                <a class="tab bg-white inline-block py-2 px-4 text-blue-500 hover:text-blue-800 font-semibold"
+                                    href="#competition-desc">Competition Desc</a>
+                            </li>
+                            <li class="mr-1">
+                                <a class="tab bg-white inline-block py-2 px-4 text-blue-500 hover:text-blue-800 font-semibold"
+                                    href="#dataset-desc">Dataset Desc</a>
+                            </li>
+                            <li class="mr-1">
+                                <a class="tab bg-white inline-block py-2 px-4 text-blue-500 hover:text-blue-800 font-semibold"
+                                    href="#submission-desc">Submission Desc</a>
+                            </li>
+                            <li class="mr-1">
+                                <a class="tab bg-white inline-block py-2 px-4 text-blue-500 hover:text-blue-800 font-semibold"
+                                    href="#rules-desc">Rules</a>
+                            </li>
+                        </ul>
+                    </div>
+                    <div id="tab-contents"
+                        class="text-xs font-normal text-left overflow-y-auto max-h-[calc(100vh-400px)] border-t border-gray-200 pt-4">
+                        <div id="config">
+                            <textarea id="config-textarea" class="w-full h-64 p-2 border rounded">Loading..</textarea>
+                            <p class="text-xs text-gray-500">Note: The config should be a valid JSON object. To learn
+                                details about entries, click <a
+                                    href="https://huggingface.co/docs/competitions/competition_repo#confjson"
+                                    target="_blank">here</a>.
+                            </p>
+                        </div>
+                        <div id="competition-desc" class="hidden">
+                            <textarea id="competition-desc-textarea"
+                                class="w-full h-64 p-2 border rounded">Loading..</textarea>
+                        </div>
+                        <div id="dataset-desc" class="hidden">
+                            <textarea id="dataset-desc-textarea"
+                                class="w-full h-64 p-2 border rounded">Loading..</textarea>
+                        </div>
+                        <div id="submission-desc" class="hidden">
+                            <textarea id="submission-desc-textarea"
+                                class="w-full h-64 p-2 border rounded">Loading..</textarea>
+                        </div>
+                        <div id="rules-desc" class="hidden">
+                            <textarea id="rules-desc-textarea"
+                                class="w-full h-64 p-2 border rounded">Loading..</textarea>
+                        </div>
+                    </div>
+                    <button id="save-button" class="mt-4 px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-700">
+                        Save
+                    </button>
+                </div>
+            </div>
+        </div>
+    </div>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/flowbite/2.2.1/flowbite.min.js"></script>
+    <script>
+        document.addEventListener("DOMContentLoaded", function () {
+            const content = document.getElementById('content');
+            const links = content.getElementsByTagName('a');
+
+            for (let i = 0; i < links.length; i++) {
+                if (!links[i].hasAttribute('target')) {
+                    links[i].setAttribute('target', '_blank');
+                }
+            }
+        });
+    </script>
+</body>
+
+<script>
+    document.addEventListener("DOMContentLoaded", function () {
+        document.querySelectorAll('.tabs a').forEach(tab => {
+            tab.addEventListener('click', event => {
+                event.preventDefault();
+                document.querySelectorAll('.tabs a').forEach(t => t.classList.remove('active'));
+                tab.classList.add('active');
+
+                document.querySelectorAll('#tab-contents > div').forEach(content => content.classList.add('hidden'));
+                const selectedTab = document.querySelector(tab.getAttribute('href'));
+                selectedTab.classList.remove('hidden');
+            });
+        });
+
+        async function fetchAdminCompInfo() {
+            const adminLoadingSpinner = document.getElementById('adminLoadingSpinner');
+            adminLoadingSpinner.classList.remove('hidden');
+            try {
+                const response = await fetch("/admin/comp_info", {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json"
+                    }
+                });
+                const data = await response.json();
+                if (response.ok) {
+                    populateAdminModal(data.response);
+                } else {
+                    alert(data.response || "Failed to fetch competition info");
+                }
+            } catch (error) {
+                console.error("Error fetching admin competition info:", error);
+                alert("An error occurred while fetching competition info.");
+            } finally {
+                adminLoadingSpinner.classList.add('hidden');
+            }
+        }
+
+        function populateAdminModal(data) {
+            document.getElementById("config-textarea").value = JSON.stringify(data.config, null, 2);
+            document.getElementById("competition-desc-textarea").value = data.markdowns["competition_desc"] || "";
+            document.getElementById("dataset-desc-textarea").value = data.markdowns["dataset_desc"] || "";
+            document.getElementById("submission-desc-textarea").value = data.markdowns["submission_desc"] || "";
+            document.getElementById("rules-desc-textarea").value = data.markdowns["rules"] || "No rules available.";
+        }
+
+        document.querySelectorAll(".tab").forEach(tab => {
+            tab.addEventListener("click", function (event) {
+                event.preventDefault();
+                const targetId = this.getAttribute("href").substring(1);
+
+                document.querySelectorAll("#tab-contents > div").forEach(content => {
+                    content.classList.add("hidden");
+                });
+                document.getElementById(targetId).classList.remove("hidden");
+
+                document.querySelectorAll(".tab").forEach(t => {
+                    t.classList.remove("text-blue-800");
+                    t.classList.add("text-blue-500");
+                });
+                this.classList.remove("text-blue-500");
+                this.classList.add("text-blue-800");
+            });
+        });
+
+        document.getElementById("admin").addEventListener("click", function () {
+            document.getElementById("admin-modal").classList.remove("hidden");
+            fetchAdminCompInfo();
+        });
+
+        document.querySelector("[data-modal-hide='admin-modal']").addEventListener("click", function () {
+            document.getElementById("admin-modal").classList.add("hidden");
+        });
+
+        document.getElementById("save-button").addEventListener("click", async function () {
+            const adminLoadingSpinner = document.getElementById('adminLoadingSpinner');
+            adminLoadingSpinner.classList.remove('hidden');
+
+            const config = document.getElementById("config-textarea").value;
+            const competitionDesc = document.getElementById("competition-desc-textarea").value;
+            const datasetDesc = document.getElementById("dataset-desc-textarea").value;
+            const submissionDesc = document.getElementById("submission-desc-textarea").value;
+            const rulesDesc = document.getElementById("rules-desc-textarea").value;
+
+            const data = {
+                config: JSON.parse(config),
+                markdowns: {
+                    competition_desc: competitionDesc,
+                    dataset_desc: datasetDesc,
+                    submission_desc: submissionDesc,
+                    rules: rulesDesc
+                }
+            };
+
+            try {
+                const response = await fetch("/admin/update_comp_info", {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json"
+                    },
+                    body: JSON.stringify(data)
+                });
+                const result = await response.json();
+                if (response.ok) {
+                    alert(result.response || "Successfully updated competition info");
+                } else {
+                    alert(result.response || "Failed to update competition info");
+                }
+            } catch (error) {
+                console.error("Error updating competition info:", error);
+                alert("An error occurred while updating competition info.");
+            } finally {
+                adminLoadingSpinner.classList.add('hidden');
+            }
+        });
+
+    });
+</script>
+
+<script>
+    document.addEventListener('DOMContentLoaded', function () {
+        document.querySelector('.confirm').addEventListener('click', function (event) {
+            event.preventDefault();
+            document.getElementById('error-message').textContent = '';
+            document.getElementById('success-message').textContent = '';
+            const loadingSpinner = document.getElementById('loadingSpinner');
+            loadingSpinner.classList.remove('hidden');
+
+            var formData = new FormData();
+            var competitionType = '{{ competition_type }}';
+
+            if (competitionType === 'generic') {
+                var submissionFile = document.getElementById('submission_file').files[0];
+                formData.append('submission_file', submissionFile);
+                formData.append('hub_model', 'None');
+            } else if (competitionType === 'script') {
+                var hubModel = document.getElementById('hub_model').value;
+                if (!hubModel) {
+                    alert('Hub model is required.');
+                    return;
+                }
+                formData.append('hub_model', hubModel);
+            } else {
+                alert('Invalid competition type.');
+                return;
+            }
+
+            var submissionComment = document.getElementById('submission_comment').value;
+            formData.append('submission_comment', submissionComment);
+
+            fetch('/new_submission', {
+                method: 'POST',
+                body: formData
+            })
+                .then(response => response.json())
+                .then(data => {
+                    loadingSpinner.classList.add('hidden');
+                    document.getElementById('success-message').textContent = data.response;
+
+                })
+                .catch((error) => {
+                    console.error('Error:', error);
+                    loadingSpinner.classList.add('hidden');
+                    document.getElementById('error-message').textContent = error;
+                });
+        });
+    });
+</script>
+
+<script>
+    function updateSelectedSubmissions() {
+        const selectedSubmissions = document.querySelectorAll('input[name="selectedSubmissions"]:checked');
+        const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+        articleLoadingSpinner.classList.remove('hidden');
+        let selectedSubmissionIds = [];
+        selectedSubmissions.forEach((submission) => {
+            selectedSubmissionIds.push(submission.value);
+        });
+
+        const updateEndpoint = '/update_selected_submissions';
+        const requestOptions = {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                "submission_ids": selectedSubmissionIds.join(',')
+            })
+        };
+
+        fetch(updateEndpoint, requestOptions)
+            .then(response => {
+                if (!response.ok) {
+                    throw new Error('Network response was not ok');
+                }
+                return response.json();
+            })
+            .then(data => {
+                if (data.success) {
+                    // Optionally, display a success message or handle accordingly
+                    console.log('Update successful');
+                    articleLoadingSpinner.classList.add('hidden');
+                } else {
+                    // Handle failure case
+                    console.log('Update failed');
+                    articleLoadingSpinner.classList.add('hidden');
+                    alert(data.error);
+                }
+                // Refresh submissions display
+                fetchAndDisplaySubmissions();
+            })
+            .catch(error => {
+                console.error('There was a problem with the fetch operation for updating:', error);
+            });
+    }
+</script>
+
+<script>
+    function updateTeamName() {
+        const teamName = document.getElementById('team_name').value;
+        const articleLoadingSpinner = document.getElementById('articleLoadingSpinner');
+        articleLoadingSpinner.classList.remove('hidden');
+
+        const updateEndpoint = '/update_team_name';
+        const requestOptions = {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                "new_team_name": teamName
+            })
+        };
+
+        fetch(updateEndpoint, requestOptions)
+            .then(response => {
+                if (!response.ok) {
+                    throw new Error('Network response was not ok');
+                }
+                return response.json();
+            })
+            .then(data => {
+                if (data.success) {
+                    // Optionally, display a success message or handle accordingly
+                    console.log('Update successful');
+                    articleLoadingSpinner.classList.add('hidden');
+                } else {
+                    // Handle failure case
+                    console.log('Update failed');
+                    articleLoadingSpinner.classList.add('hidden');
+                    alert(data.error);
+                }
+                // Refresh submissions display
+                fetchAndDisplaySubmissions();
+            })
+            .catch(error => {
+                console.error('There was a problem with the fetch operation for updating:', error);
+            });
+    }
+</script>
+
+<script>
+    function showAdminModal() {
+        const modal = document.getElementById('admin-modal');
+        modal.classList.add('flex');
+        modal.classList.remove('hidden');
+    }
+
+    function hideAdminModal() {
+        const modal = document.getElementById('admin-modal');
+        modal.classList.remove('flex');
+        modal.classList.add('hidden');
+    }
+
+    document.querySelector('#admin').addEventListener('click', function () {
+        showAdminModal();
+    });
+
+    document.querySelector('[data-modal-hide="admin-modal"]').addEventListener('click', function () {
+        hideAdminModal();
+    });
+</script>
+
+</html>

competitions/tests/test_dummy.py

@@ -0,0 +1,2 @@
+def test_dummy():
+    assert 1 + 1 == 2

competitions/text.py

@@ -0,0 +1,17 @@
+SUBMISSION_TEXT = """### My Submissions\n\nYou can make upto {} submissions per day.
+The test data has been divided into public and private splits.
+Your score on the public split will be shown on the leaderboard.
+Your final score will be based on your private split performance.
+The final rankings will be based on the private split performance.
+"""
+
+SUBMISSION_ERROR = """Submission is not in a proper format.
+Please check evaluation instructions for more details."""
+
+SUBMISSION_SELECTION_TEXT = """\n\nYou can select upto {} submissions for private leaderboard."""
+
+NO_SUBMISSIONS = """You have not made any submissions yet."""
+
+SUBMISSION_SUCCESS = """Submission successful! You have {} submissions remaining today."""
+
+SUBMISSION_LIMIT_REACHED = """You have reached your submission limit for today. Please try again tomorrow."""

competitions/utils.py

@@ -0,0 +1,386 @@
+import io
+import json
+import os
+import shlex
+import subprocess
+import traceback
+
+import requests
+from fastapi import Request
+from huggingface_hub import HfApi, hf_hub_download
+from loguru import logger
+
+from competitions.enums import SubmissionStatus
+from competitions.params import EvalParams
+
+from . import HF_URL
+
+
+USER_TOKEN = os.environ.get("USER_TOKEN")
+
+
+def token_information(token):
+    if token.startswith("hf_oauth"):
+        _api_url = HF_URL + "/oauth/userinfo"
+    else:
+        _api_url = HF_URL + "/api/whoami-v2"
+    headers = {}
+    cookies = {}
+    if token.startswith("hf_"):
+        headers["Authorization"] = f"Bearer {token}"
+    else:
+        cookies = {"token": token}
+    try:
+        response = requests.get(
+            _api_url,
+            headers=headers,
+            cookies=cookies,
+            timeout=3,
+        )
+    except (requests.Timeout, ConnectionError) as err:
+        logger.error(f"Failed to request whoami-v2 - {repr(err)}")
+        raise Exception("Hugging Face Hub is unreachable, please try again later.")
+
+    if response.status_code != 200:
+        logger.error(f"Failed to request whoami-v2 - {response.status_code}")
+        raise Exception("Invalid token.")
+
+    resp = response.json()
+    user_info = {}
+
+    if token.startswith("hf_oauth"):
+        user_info["id"] = resp["sub"]
+        user_info["name"] = resp["preferred_username"]
+        user_info["orgs"] = [resp["orgs"][k]["preferred_username"] for k in range(len(resp["orgs"]))]
+    else:
+        user_info["id"] = resp["id"]
+        user_info["name"] = resp["name"]
+        user_info["orgs"] = [resp["orgs"][k]["name"] for k in range(len(resp["orgs"]))]
+    return user_info
+
+
+def user_authentication(request: Request):
+    auth_header = request.headers.get("Authorization")
+    bearer_token = None
+
+    if auth_header and auth_header.startswith("Bearer "):
+        bearer_token = auth_header.split(" ")[1]
+
+    if bearer_token:
+        try:
+            _ = token_information(token=bearer_token)
+            return bearer_token
+        except Exception as e:
+            logger.error(f"Failed to verify token: {e}")
+            return None
+
+    if USER_TOKEN is not None:
+        try:
+            _ = token_information(token=USER_TOKEN)
+            return USER_TOKEN
+        except Exception as e:
+            logger.error(f"Failed to verify token: {e}")
+            return None
+
+    if "oauth_info" in request.session:
+        try:
+            _ = token_information(token=request.session["oauth_info"]["access_token"])
+            return request.session["oauth_info"]["access_token"]
+        except Exception as e:
+            request.session.pop("oauth_info", None)
+            logger.error(f"Failed to verify token: {e}")
+            return None
+
+    return None
+
+
+def user_authentication_dep(token, return_raw=False):
+    if token.startswith("hf_oauth"):
+        _api_url = HF_URL + "/oauth/userinfo"
+    else:
+        _api_url = HF_URL + "/api/whoami-v2"
+    headers = {}
+    cookies = {}
+    if token.startswith("hf_"):
+        headers["Authorization"] = f"Bearer {token}"
+    else:
+        cookies = {"token": token}
+    try:
+        response = requests.get(
+            _api_url,
+            headers=headers,
+            cookies=cookies,
+            timeout=3,
+        )
+    except (requests.Timeout, ConnectionError) as err:
+        logger.error(f"Failed to request whoami-v2 - {repr(err)}")
+        raise Exception("Hugging Face Hub is unreachable, please try again later.")
+
+    resp = response.json()
+    if return_raw:
+        return resp
+
+    user_info = {}
+    if "error" in resp:
+        return resp
+    if token.startswith("hf_oauth"):
+        user_info["id"] = resp["sub"]
+        user_info["name"] = resp["preferred_username"]
+        user_info["orgs"] = [resp["orgs"][k]["preferred_username"] for k in range(len(resp["orgs"]))]
+    else:
+
+        user_info["id"] = resp["id"]
+        user_info["name"] = resp["name"]
+        user_info["orgs"] = [resp["orgs"][k]["name"] for k in range(len(resp["orgs"]))]
+    return user_info
+
+
+def make_clickable_user(user_id):
+    link = "https://huggingface.co/" + user_id
+    return f'<a  target="_blank" href="{link}">{user_id}</a>'
+
+
+def run_evaluation(params, local=False, wait=False):
+    params = json.loads(params)
+    if isinstance(params, str):
+        params = json.loads(params)
+    params = EvalParams(**params)
+    if not local:
+        params.output_path = "/tmp/model"
+    params.save(output_dir=params.output_path)
+    cmd = [
+        "python",
+        "-m",
+        "competitions.evaluate",
+        "--config",
+        os.path.join(params.output_path, "params.json"),
+    ]
+
+    cmd = [str(c) for c in cmd]
+    logger.info(cmd)
+    env = os.environ.copy()
+    cmd = shlex.split(" ".join(cmd))
+    process = subprocess.Popen(cmd, env=env)
+    if wait:
+        process.wait()
+    return process.pid
+
+
+def pause_space(params):
+    if "SPACE_ID" in os.environ:
+        if os.environ["SPACE_ID"].split("/")[-1].startswith("comp-"):
+            logger.info("Pausing space...")
+            api = HfApi(token=params.token)
+            api.pause_space(repo_id=os.environ["SPACE_ID"])
+
+
+def delete_space(params):
+    if "SPACE_ID" in os.environ:
+        if os.environ["SPACE_ID"].split("/")[-1].startswith("comp-"):
+            logger.info("Deleting space...")
+            api = HfApi(token=params.token)
+            api.delete_repo(repo_id=os.environ["SPACE_ID"], repo_type="space")
+
+
+def download_submission_info(params):
+    user_fname = hf_hub_download(
+        repo_id=params.competition_id,
+        filename=f"submission_info/{params.team_id}.json",
+        token=params.token,
+        repo_type="dataset",
+    )
+    with open(user_fname, "r", encoding="utf-8") as f:
+        user_submission_info = json.load(f)
+
+    return user_submission_info
+
+
+def upload_submission_info(params, user_submission_info):
+    user_submission_info_json = json.dumps(user_submission_info, indent=4)
+    user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
+    user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+    api = HfApi(token=params.token)
+    api.upload_file(
+        path_or_fileobj=user_submission_info_json_buffer,
+        path_in_repo=f"submission_info/{params.team_id}.json",
+        repo_id=params.competition_id,
+        repo_type="dataset",
+    )
+
+
+def update_submission_status(params, status):
+    user_submission_info = download_submission_info(params)
+    for submission in user_submission_info["submissions"]:
+        if submission["submission_id"] == params.submission_id:
+            submission["status"] = status
+            break
+    upload_submission_info(params, user_submission_info)
+
+
+def update_submission_score(params, public_score, private_score):
+    user_submission_info = download_submission_info(params)
+    for submission in user_submission_info["submissions"]:
+        if submission["submission_id"] == params.submission_id:
+            submission["public_score"] = public_score
+            submission["private_score"] = private_score
+            submission["status"] = "done"
+            break
+    upload_submission_info(params, user_submission_info)
+
+
+def monitor(func):
+    def wrapper(*args, **kwargs):
+        params = kwargs.get("params", None)
+        if params is None and len(args) > 0:
+            params = args[0]
+
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            error_message = f"""{func.__name__} has failed due to an exception: {traceback.format_exc()}"""
+            logger.error(error_message)
+            logger.error(str(e))
+            update_submission_status(params, SubmissionStatus.FAILED.value)
+            pause_space(params)
+
+    return wrapper
+
+
+def uninstall_requirements(requirements_fname):
+    if os.path.exists(requirements_fname):
+        # read the requirements.txt
+        uninstall_list = []
+        with open(requirements_fname, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.startswith("-"):
+                    uninstall_list.append(line[1:])
+
+        # create an uninstall.txt
+        with open("uninstall.txt", "w", encoding="utf-8") as f:
+            for line in uninstall_list:
+                f.write(line)
+
+        pipe = subprocess.Popen(
+            [
+                "pip",
+                "uninstall",
+                "-r",
+                "uninstall.txt",
+                "-y",
+            ],
+        )
+        pipe.wait()
+        logger.info("Requirements uninstalled.")
+        return
+
+
+def install_requirements(requirements_fname):
+    # check if params.project_name has a requirements.txt
+    if os.path.exists(requirements_fname):
+        # install the requirements using subprocess, wait for it to finish
+        install_list = []
+
+        with open(requirements_fname, "r", encoding="utf-8") as f:
+            for line in f:
+                # if line startswith - then skip but dont skip if line startswith --
+                if line.startswith("-"):
+                    if not line.startswith("--"):
+                        continue
+                install_list.append(line)
+
+        with open("install.txt", "w", encoding="utf-8") as f:
+            for line in install_list:
+                f.write(line)
+
+        pipe = subprocess.Popen(
+            [
+                "pip",
+                "install",
+                "-r",
+                "install.txt",
+            ],
+        )
+        pipe.wait()
+        logger.info("Requirements installed.")
+        return
+    logger.info("No requirements.txt found. Skipping requirements installation.")
+    return
+
+
+def is_user_admin(user_token, competition_organization):
+    user_info = token_information(token=user_token)
+    user_orgs = user_info.get("orgs", [])
+    for org in user_orgs:
+        if org == competition_organization:
+            return True
+    return False
+
+
+def get_team_name(user_token, competition_id, hf_token):
+    user_info = token_information(token=user_token)
+    user_id = user_info["id"]
+    user_team = hf_hub_download(
+        repo_id=competition_id,
+        filename="user_team.json",
+        token=hf_token,
+        repo_type="dataset",
+    )
+    with open(user_team, "r", encoding="utf-8") as f:
+        user_team = json.load(f)
+
+    if user_id not in user_team:
+        return None
+
+    team_id = user_team[user_id]
+
+    team_metadata = hf_hub_download(
+        repo_id=competition_id,
+        filename="teams.json",
+        token=hf_token,
+        repo_type="dataset",
+    )
+    with open(team_metadata, "r", encoding="utf-8") as f:
+        team_metadata = json.load(f)
+
+    team_name = team_metadata[team_id]["name"]
+    return team_name
+
+
+def update_team_name(user_token, new_team_name, competition_id, hf_token):
+    user_info = token_information(token=user_token)
+    user_id = user_info["id"]
+    user_team = hf_hub_download(
+        repo_id=competition_id,
+        filename="user_team.json",
+        token=hf_token,
+        repo_type="dataset",
+    )
+    with open(user_team, "r", encoding="utf-8") as f:
+        user_team = json.load(f)
+
+    if user_id not in user_team:
+        raise Exception("User is not part of a team")
+
+    team_id = user_team[user_id]
+
+    team_metadata = hf_hub_download(
+        repo_id=competition_id,
+        filename="teams.json",
+        token=hf_token,
+        repo_type="dataset",
+    )
+    with open(team_metadata, "r", encoding="utf-8") as f:
+        team_metadata = json.load(f)
+
+    team_metadata[team_id]["name"] = new_team_name
+    team_metadata_json = json.dumps(team_metadata, indent=4)
+    team_metadata_json_bytes = team_metadata_json.encode("utf-8")
+    team_metadata_json_buffer = io.BytesIO(team_metadata_json_bytes)
+    api = HfApi(token=hf_token)
+    api.upload_file(
+        path_or_fileobj=team_metadata_json_buffer,
+        path_in_repo="teams.json",
+        repo_id=competition_id,
+        repo_type="dataset",
+    )
+    return new_team_name