import gradio as gr
from transformers import AutoTokenizer

# Load a tokenizer
tokenizer = AutoTokenizer.from_pretrained("otmanheddouch/shakespear-tokenizer")

# Define function that returns a dictionary
def tokenize_text(text):
    tokens = tokenizer.tokenize(text)
    token_ids = tokenizer.convert_tokens_to_ids(tokens)
    return {
        "tokens": tokens,
        "token_ids": token_ids
    }

# Create Gradio interface
demo = gr.Interface(
    fn=tokenize_text,
    inputs=gr.Textbox(label="Enter Text"),
    outputs=gr.JSON(label="Token Output"),
    title="Text Tokenizer",
    description="Enter some text to see its tokens and token IDs."
)

# Launch the app
demo.launch()