Update TokenViz: AutoTokenizer Visualization Tool

README.md

@@ -1,14 +1,14 @@
 ---
-title: Tokenizers
-emoji: 😻
+title: "TokenViz: AutoTokenizer Visualization Tool"
+emoji: 🔍
 colorFrom: green
-colorTo: green
+colorTo: indigo
 sdk: streamlit
 sdk_version: 1.39.0
 app_file: app.py
 pinned: false
 license: mit
-short_description: AutoTokenizerでのtoken化の結果を取得
+short_description: Visualize the results of AutoTokenizer
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -17,6 +17,7 @@ DEFAULT_TOKENIZER_NAME = os.environ.get(
 DEFAULT_TEXT = """
 hello world!
 こんにちは、世界！
+你好,世界
 """.strip()
 
 DEFAULT_COLOR = "gray"
@@ -26,13 +27,13 @@ COLORS_CYCLE = [
 ]
 
 
-def color_cycle_generater():
-    def _color_cycle_generater():
+def color_cycle_generator():
+    def _color_cycle_generator():
         while True:
             for color in COLORS_CYCLE:
                 yield color
 
-    return _color_cycle_generater()
+    return _color_cycle_generator()
 
 
 @st.cache_resource
@@ -43,18 +44,20 @@ def get_tokenizer(tokenizer_name: str = DEFAULT_TOKENIZER_NAME):
 
 def main():
     st.set_page_config(
-        page_title="tokenizer visualizer",
+        page_title="TokenViz: AutoTokenizer Visualization Tool",
         layout="centered",
         initial_sidebar_state="auto",
     )
 
     st.title("tokenizer visualizer")
-    st.text_input("tokenizer name", key="tokenizer_name", value=DEFAULT_TOKENIZER_NAME)
+    st.text_input(
+        "AutoTokenizer model name", key="tokenizer_name", value=DEFAULT_TOKENIZER_NAME
+    )
     if st.session_state.tokenizer_name:
         tokenizer = get_tokenizer(st.session_state.tokenizer_name)
     st.text_input("subword prefix", key="subword_prefix", value="##")
     st.text_area("text", key="text", height=200, value=DEFAULT_TEXT)
-    # submit
+    # Submit
     if st.button("tokenize"):
         text = st.session_state.text.strip()
         subword_prefix = st.session_state.subword_prefix.strip()
@@ -72,13 +75,13 @@ def main():
         st.markdown(f"total tokens: **{total_tokens}**")
         tab_main, tab_token_table = st.tabs(["tokens", "table"])
 
-        color_gen = color_cycle_generater()
+        color_gen = color_cycle_generator()
         with tab_main:
             current_subword_color = next(color_gen)
             token_html = ""
             for idx, (token_id, token) in enumerate(zip(token_ids, tokens)):
                 if len(subword_prefix) == 0:
-                    target_border = f"1px solid {DEFAULT_COLOR}"
+                    token_border = f"1px solid {DEFAULT_COLOR}"
                 else:
                     current_token_is_subword = token.startswith(subword_prefix)
                     next_token_is_subword = idx + 1 < total_tokens and tokens[
@@ -89,12 +92,12 @@ def main():
                         current_subword_color = next(color_gen)
 
                     if current_token_is_subword or next_token_is_subword:
-                        target_border = f"1px solid {current_subword_color}"
+                        token_border = f"1px solid {current_subword_color}"
                     else:
-                        target_border = f"1px solid {DEFAULT_COLOR}"
+                        token_border = f"1px solid {DEFAULT_COLOR}"
 
                 html_escaped_token = token.replace("<", "&lt;").replace(">", "&gt;")
-                token_html += f'<span title="{str(token_id)}" style="border: {target_border}; border-radius: 3px; padding: 2px; margin: 2px;">{html_escaped_token}</span>'
+                token_html += f'<span title="{str(token_id)}" style="border: {token_border}; border-radius: 3px; padding: 2px; margin: 2px;">{html_escaped_token}</span>'
             st.html(
                 f"<p style='line-height:2em;'>{token_html}</p>",
             )