Textbox tokenizer
## Installation
```bash
pip install gradio_tokenizertextbox
```
## Usage
```python
#
# demo/app.py
#
import gradio as gr
from gradio_tokenizertextbox import TokenizerTextBox
import json
# --- Data and Helper Functions ---
TOKENIZER_OPTIONS = {
"Xenova/clip-vit-large-patch14": "CLIP ViT-L/14",
"Xenova/gpt-4": "gpt-4 / gpt-3.5-turbo / text-embedding-ada-002",
"Xenova/text-davinci-003": "text-davinci-003 / text-davinci-002",
"Xenova/gpt-3": "gpt-3",
"Xenova/grok-1-tokenizer": "Grok-1",
"Xenova/claude-tokenizer": "Claude",
"Xenova/mistral-tokenizer-v3": "Mistral v3",
"Xenova/mistral-tokenizer-v1": "Mistral v1",
"Xenova/gemma-tokenizer": "Gemma",
"Xenova/llama-3-tokenizer": "Llama 3",
"Xenova/llama-tokenizer": "LLaMA / Llama 2",
"Xenova/c4ai-command-r-v01-tokenizer": "Cohere Command-R",
"Xenova/t5-small": "T5",
"Xenova/bert-base-cased": "bert-base-cased",
}
dropdown_choices = [
(display_name, model_name)
for model_name, display_name in TOKENIZER_OPTIONS.items()
]
def process_output(tokenization_data):
"""
This function receives the full dictionary from the component.
"""
if not tokenization_data:
return {"status": "Waiting for input..."}
return tokenization_data
# --- Gradio Application ---
with gr.Blocks(theme=gr.themes.Soft()) as demo:
# --- Header and Information ---
gr.Markdown("# TokenizerTextBox Component Demo")
gr.Markdown("Component idea taken from the original example application on [Xenova Tokenizer Playground](https://github.com/huggingface/transformers.js-examples/tree/main/the-tokenizer-playground)")
# --- Global Controls (affect both tabs) ---
with gr.Row():
model_selector = gr.Dropdown(
label="Select a Tokenizer",
choices=dropdown_choices,
value="Xenova/clip-vit-large-patch14",
)
display_mode_radio = gr.Radio(
["text", "token_ids", "hidden"],
label="Display Mode",
value="text"
)
# --- Tabbed Interface for Different Modes ---
with gr.Tabs():
# --- Tab 1: Standalone Mode ---
with gr.TabItem("Standalone Mode"):
gr.Markdown("### In this mode, the component acts as its own interactive textbox.")
gr.Markdown("💻 GitHub Code")
standalone_tokenizer = TokenizerTextBox(
label="Type your text here",
value="Gradio is an awesome tool for building ML demos!",
model="Xenova/clip-vit-large-patch14",
display_mode="text",
preview_tokens=True
)
standalone_output = gr.JSON(label="Component Output")
standalone_tokenizer.change(process_output, standalone_tokenizer, standalone_output)
# --- Tab 2: Listener ("Push") Mode ---
with gr.TabItem("Listener Mode"):
gr.Markdown("### In this mode, the component is a read-only visualizer for other text inputs.")
with gr.Row():
prompt_1 = gr.Textbox(label="Prompt Part 1", value="A photorealistic image of an astronaut")
prompt_2 = gr.Textbox(label="Prompt Part 2", value="riding a horse on Mars")
visualizer = TokenizerTextBox(
label="Concatenated Prompt Visualization",
hide_input=True, # Hides the internal textbox
model="Xenova/clip-vit-large-patch14",
display_mode="text",
)
visualizer_output = gr.JSON(label="Visualizer Component Output")
# --- "Push" Logic ---
def update_visualizer_text(p1, p2):
concatenated_text = f"{p1}, {p2}"
# Return a new value for the visualizer.
# The postprocess method will correctly handle this string.
return gr.update(value=concatenated_text)
# Listen for changes on the source textboxes
prompt_1.change(update_visualizer_text, [prompt_1, prompt_2], visualizer)
prompt_2.change(update_visualizer_text, [prompt_1, prompt_2], visualizer)
# Also connect the visualizer to its own JSON output
visualizer.change(process_output, visualizer, visualizer_output)
# Run once on load to show the initial state
demo.load(update_visualizer_text, [prompt_1, prompt_2], visualizer)
# --- Link Global Controls to Both Components ---
# Create a list of all TokenizerTextBox components that need to be updated
all_tokenizers = [standalone_tokenizer, visualizer]
model_selector.change(
fn=lambda model: [gr.update(model=model) for _ in all_tokenizers],
inputs=model_selector,
outputs=all_tokenizers
)
display_mode_radio.change(
fn=lambda mode: [gr.update(display_mode=mode) for _ in all_tokenizers],
inputs=display_mode_radio,
outputs=all_tokenizers
)
if __name__ == '__main__':
demo.launch()
```
## `TokenizerTextBox`
### Initialization
| name | type | default | description |
|---|---|---|---|
value |
```python typing.Union[str, dict, typing.Callable, NoneType][ str, dict, Callable, None ] ``` | None |
The initial value. Can be a string to initialize the text, or a dictionary for full state. If a function is provided, it will be called when the app loads to set the initial value. |
model |
```python str ``` | "Xenova/gpt-3" |
The name of a Hugging Face tokenizer to use (must be compatible with Transformers.js). Defaults to "Xenova/gpt-2". |
display_mode |
```python "text" | "token_ids" | "hidden" ``` | "text" |
Controls the content of the token visualization panel. Can be 'text' (default), 'token_ids', or 'hidden'. |
hide_input |
```python bool ``` | False |
If True, the component's own textbox is hidden, turning it into a read-only visualizer. Defaults to False. |
model_max_length |
```python int | None ``` | None |
The maximum number of tokens for the model. If the token count exceeds this, the counter will turn red. If not provided, the component will try to detect it from the loaded tokenizer. |
preview_tokens |
```python bool ``` | False |
If True, the component displays the formatted tokens. |
lines |
```python int ``` | 2 |
The minimum number of line rows for the textarea. |
max_lines |
```python int | None ``` | None |
The maximum number of line rows for the textarea. |
placeholder |
```python str | None ``` | None |
A placeholder hint to display in the textarea when it is empty. |
autofocus |
```python bool ``` | False |
If True, will focus on the textbox when the page loads. |
autoscroll |
```python bool ``` | True |
If True, will automatically scroll to the bottom of the textbox when the value changes. |
text_align |
```python typing.Optional[typing.Literal["left", "right"]][ "left" | "right", None ] ``` | None |
How to align the text in the textbox, can be: "left" or "right". |
rtl |
```python bool ``` | False |
If True, sets the direction of the text to right-to-left. |
show_copy_button |
```python bool ``` | False |
If True, a copy button will be shown. |
max_length |
```python int | None ``` | None |
The maximum number of characters allowed in the textbox. |
label |
```python str | None ``` | None |
The label for this component, displayed above the component. |
info |
```python str | None ``` | None |
Additional component description, displayed below the label. |
every |
```python float | None ``` | None |
If `value` is a callable, this sets a timer to run the function repeatedly. |
show_label |
```python bool ``` | True |
If False, the label is not displayed. |
container |
```python bool ``` | True |
If False, the component will not be wrapped in a container. |
scale |
```python int | None ``` | None |
The relative size of the component compared to others in a `gr.Row` or `gr.Column`. |
min_width |
```python int ``` | 160 |
The minimum-width of the component in pixels. |
interactive |
```python bool | None ``` | None |
If False, the user will not be able to edit the text. |
visible |
```python bool ``` | True |
If False, the component will be hidden. |
elem_id |
```python str | None ``` | None |
An optional string that is assigned as the id of this component in the HTML DOM. |
elem_classes |
```python list[str] | str | None ``` | None |
An optional list of strings that are assigned as the classes of this component in the HTML DOM. |