We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Error when "--field -5" is passed to espnet2.bin.tokenize_text
**Describe the bug**

```
D:\Anaconda\python.exe D:/repos/espnet/espnet2/bin/tokenize_text.py --token_type phn --input tmp.txt --output tmp.phn --field -2 --cleaner none --g2p g2p_en --add_symbol '$<blank>:0' --add_symbol '<oov>:1' --add_symbol '<sos/eos>:-1' --write_vocabulary false --keep_all_fields true
Traceback (most recent call last):
  File "D:/repos/espnet/espnet2/bin/tokenize_text.py", line 297, in <module>
    main()
  File "D:/repos/espnet/espnet2/bin/tokenize_text.py", line 293, in main
    tokenize(**kwargs)
  File "D:/repos/espnet/espnet2/bin/tokenize_text.py", line 112, in tokenize
    field = field2slice(field)
  File "D:/repos/espnet/espnet2/bin/tokenize_text.py", line 59, in field2slice
    slic = slice(s1 - 1, s2)
TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'
```
This is because of a missing None check [here](https://github.com/espnet/espnet/blob/master/espnet2/bin/tokenize_text.py#L59)


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `espnet2/bin/tokenize_text.py`
Content:
```
1 #!/usr/bin/env python3
2 import argparse
3 from collections import Counter
4 import logging
5 from pathlib import Path
6 import sys
7 from typing import List
8 from typing import Optional
9 
10 from typeguard import check_argument_types
11 
12 from espnet.utils.cli_utils import get_commandline_args
13 from espnet2.text.build_tokenizer import build_tokenizer
14 from espnet2.text.cleaner import TextCleaner
15 from espnet2.utils.types import str2bool
16 from espnet2.utils.types import str_or_none
17 
18 
19 def field2slice(field: Optional[str]) -> slice:
20     """Convert field string to slice
21 
22     Note that field string accepts 1-based integer.
23 
24     Examples:
25         >>> field2slice("1-")
26         slice(0, None, None)
27         >>> field2slice("1-3")
28         slice(0, 3, None)
29         >>> field2slice("-3")
30         slice(None, 3, None)
31 
32     """
33     field = field.strip()
34     try:
35         if "-" in field:
36             # e.g. "2-" or "2-5" or "-7"
37             s1, s2 = field.split("-", maxsplit=1)
38             if s1.strip() == "":
39                 s1 = None
40             else:
41                 s1 = int(s1)
42                 if s1 == 0:
43                     raise ValueError("1-based string")
44             if s2.strip() == "":
45                 s2 = None
46             else:
47                 s2 = int(s2)
48         else:
49             # e.g. "2"
50             s1 = int(field)
51             s2 = s1 + 1
52             if s1 == 0:
53                 raise ValueError("must be 1 or more value")
54     except ValueError:
55         raise RuntimeError(f"Format error: e.g. '2-', '2-5', or '-5': {field}")
56 
57     # -1 because of 1-based integer following "cut" command
58     # e.g "1-3" -> slice(0, 3)
59     slic = slice(s1 - 1, s2)
60     return slic
61 
62 
63 def tokenize(
64     input: str,
65     output: str,
66     field: Optional[str],
67     delimiter: Optional[str],
68     token_type: str,
69     space_symbol: str,
70     non_linguistic_symbols: Optional[str],
71     bpemodel: Optional[str],
72     log_level: str,
73     write_vocabulary: bool,
74     vocabulary_size: int,
75     remove_non_linguistic_symbols: bool,
76     cutoff: int,
77     add_symbol: List[str],
78     cleaner: Optional[str],
79     g2p: Optional[str],
80 ):
81     assert check_argument_types()
82 
83     logging.basicConfig(
84         level=log_level,
85         format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
86     )
87     if input == "-":
88         fin = sys.stdin
89     else:
90         fin = Path(input).open("r", encoding="utf-8")
91     if output == "-":
92         fout = sys.stdout
93     else:
94         p = Path(output)
95         p.parent.mkdir(parents=True, exist_ok=True)
96         fout = p.open("w", encoding="utf-8")
97 
98     cleaner = TextCleaner(cleaner)
99     tokenizer = build_tokenizer(
100         token_type=token_type,
101         bpemodel=bpemodel,
102         delimiter=delimiter,
103         space_symbol=space_symbol,
104         non_linguistic_symbols=non_linguistic_symbols,
105         remove_non_linguistic_symbols=remove_non_linguistic_symbols,
106         g2p_type=g2p,
107     )
108 
109     counter = Counter()
110     if field is not None:
111         field = field2slice(field)
112 
113     for line in fin:
114         line = line.rstrip()
115         if field is not None:
116             # e.g. field="2-"
117             # uttidA hello world!! -> hello world!!
118             tokens = line.split(delimiter)
119             tokens = tokens[field]
120             if delimiter is None:
121                 line = " ".join(tokens)
122             else:
123                 line = delimiter.join(tokens)
124 
125         line = cleaner(line)
126         tokens = tokenizer.text2tokens(line)
127         if not write_vocabulary:
128             fout.write(" ".join(tokens) + "\n")
129         else:
130             for t in tokens:
131                 counter[t] += 1
132 
133     if not write_vocabulary:
134         return
135 
136     # ======= write_vocabulary mode from here =======
137     # Sort by the number of occurrences in descending order
138     # and filter lower frequency words than cutoff value
139     words_and_counts = list(
140         filter(lambda x: x[1] > cutoff, sorted(counter.items(), key=lambda x: -x[1]))
141     )
142     # Restrict the vocabulary size
143     if vocabulary_size > 0:
144         if vocabulary_size < len(add_symbol):
145             raise RuntimeError(f"vocabulary_size is too small: {vocabulary_size}")
146         words_and_counts = words_and_counts[: vocabulary_size - len(add_symbol)]
147 
148     # Parse the values of --add_symbol
149     for symbol_and_id in add_symbol:
150         # e.g symbol="<blank>:0"
151         try:
152             symbol, idx = symbol_and_id.split(":")
153             idx = int(idx)
154         except ValueError:
155             raise RuntimeError(f"Format error: e.g. '<blank>:0': {symbol_and_id}")
156         symbol = symbol.strip()
157 
158         # e.g. idx=0  -> append as the first symbol
159         # e.g. idx=-1 -> append as the last symbol
160         if idx < 0:
161             idx = len(words_and_counts) + 1 + idx
162         words_and_counts.insert(idx, (symbol, None))
163 
164     # Write words
165     for w, c in words_and_counts:
166         fout.write(w + "\n")
167 
168     # Logging
169     total_count = sum(counter.values())
170     invocab_count = sum(c for w, c in words_and_counts if c is not None)
171     logging.info(f"OOV rate = {(total_count - invocab_count) / total_count * 100} %")
172 
173 
174 def get_parser() -> argparse.ArgumentParser:
175     parser = argparse.ArgumentParser(
176         description="Tokenize texts",
177         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
178     )
179     parser.add_argument(
180         "--log_level",
181         type=lambda x: x.upper(),
182         default="INFO",
183         choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
184         help="The verbose level of logging",
185     )
186 
187     parser.add_argument(
188         "--input", "-i", required=True, help="Input text. - indicates sys.stdin"
189     )
190     parser.add_argument(
191         "--output", "-o", required=True, help="Output text. - indicates sys.stdout"
192     )
193     parser.add_argument(
194         "--field",
195         "-f",
196         help="The target columns of the input text as 1-based integer. e.g 2-",
197     )
198     parser.add_argument(
199         "--token_type",
200         "-t",
201         default="char",
202         choices=["char", "bpe", "word", "phn"],
203         help="Token type",
204     )
205     parser.add_argument("--delimiter", "-d", default=None, help="The delimiter")
206     parser.add_argument("--space_symbol", default="<space>", help="The space symbol")
207     parser.add_argument("--bpemodel", default=None, help="The bpemodel file path")
208     parser.add_argument(
209         "--non_linguistic_symbols",
210         type=str_or_none,
211         help="non_linguistic_symbols file path",
212     )
213     parser.add_argument(
214         "--remove_non_linguistic_symbols",
215         type=str2bool,
216         default=False,
217         help="Remove non-language-symbols from tokens",
218     )
219     parser.add_argument(
220         "--cleaner",
221         type=str_or_none,
222         choices=[None, "tacotron", "jaconv", "vietnamese"],
223         default=None,
224         help="Apply text cleaning",
225     )
226     parser.add_argument(
227         "--g2p",
228         type=str_or_none,
229         choices=[
230             None,
231             "g2p_en",
232             "g2p_en_no_space",
233             "pyopenjtalk",
234             "pyopenjtalk_kana",
235             "pyopenjtalk_accent",
236             "pyopenjtalk_accent_with_pause",
237             "pypinyin_g2p",
238             "pypinyin_g2p_phone",
239             "espeak_ng_arabic",
240         ],
241         default=None,
242         help="Specify g2p method if --token_type=phn",
243     )
244 
245     group = parser.add_argument_group("write_vocabulary mode related")
246     group.add_argument(
247         "--write_vocabulary",
248         type=str2bool,
249         default=False,
250         help="Write tokens list instead of tokenized text per line",
251     )
252     group.add_argument("--vocabulary_size", type=int, default=0, help="Vocabulary size")
253     group.add_argument(
254         "--cutoff",
255         default=0,
256         type=int,
257         help="cut-off frequency used for write-vocabulary mode",
258     )
259     group.add_argument(
260         "--add_symbol",
261         type=str,
262         default=[],
263         action="append",
264         help="Append symbol e.g. --add_symbol '<blank>:0' --add_symbol '<unk>:1'",
265     )
266 
267     return parser
268 
269 
270 def main(cmd=None):
271     print(get_commandline_args(), file=sys.stderr)
272     parser = get_parser()
273     args = parser.parse_args(cmd)
274     kwargs = vars(args)
275     tokenize(**kwargs)
276 
277 
278 if __name__ == "__main__":
279     main()
280
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/espnet2/bin/tokenize_text.py b/espnet2/bin/tokenize_text.py
--- a/espnet2/bin/tokenize_text.py
+++ b/espnet2/bin/tokenize_text.py
@@ -28,7 +28,6 @@
         slice(0, 3, None)
         >>> field2slice("-3")
         slice(None, 3, None)
-
     """
     field = field.strip()
     try:
@@ -54,9 +53,12 @@
     except ValueError:
         raise RuntimeError(f"Format error: e.g. '2-', '2-5', or '-5': {field}")
 
-    # -1 because of 1-based integer following "cut" command
-    # e.g "1-3" -> slice(0, 3)
-    slic = slice(s1 - 1, s2)
+    if s1 is None:
+        slic = slice(None, s2)
+    else:
+        # -1 because of 1-based integer following "cut" command
+        # e.g "1-3" -> slice(0, 3)
+        slic = slice(s1 - 1, s2)
     return slic

{"golden_diff": "diff --git a/espnet2/bin/tokenize_text.py b/espnet2/bin/tokenize_text.py\n--- a/espnet2/bin/tokenize_text.py\n+++ b/espnet2/bin/tokenize_text.py\n@@ -28,7 +28,6 @@\n         slice(0, 3, None)\n         >>> field2slice(\"-3\")\n         slice(None, 3, None)\n-\n     \"\"\"\n     field = field.strip()\n     try:\n@@ -54,9 +53,12 @@\n     except ValueError:\n         raise RuntimeError(f\"Format error: e.g. '2-', '2-5', or '-5': {field}\")\n \n-    # -1 because of 1-based integer following \"cut\" command\n-    # e.g \"1-3\" -> slice(0, 3)\n-    slic = slice(s1 - 1, s2)\n+    if s1 is None:\n+        slic = slice(None, s2)\n+    else:\n+        # -1 because of 1-based integer following \"cut\" command\n+        # e.g \"1-3\" -> slice(0, 3)\n+        slic = slice(s1 - 1, s2)\n     return slic\n", "issue": "Error when \"--field -5\" is passed to espnet2.bin.tokenize_text\n**Describe the bug**\r\n\r\n```\r\nD:\\Anaconda\\python.exe D:/repos/espnet/espnet2/bin/tokenize_text.py --token_type phn --input tmp.txt --output tmp.phn --field -2 --cleaner none --g2p g2p_en --add_symbol '$<blank>:0' --add_symbol '<oov>:1' --add_symbol '<sos/eos>:-1' --write_vocabulary false --keep_all_fields true\r\nTraceback (most recent call last):\r\n  File \"D:/repos/espnet/espnet2/bin/tokenize_text.py\", line 297, in <module>\r\n    main()\r\n  File \"D:/repos/espnet/espnet2/bin/tokenize_text.py\", line 293, in main\r\n    tokenize(**kwargs)\r\n  File \"D:/repos/espnet/espnet2/bin/tokenize_text.py\", line 112, in tokenize\r\n    field = field2slice(field)\r\n  File \"D:/repos/espnet/espnet2/bin/tokenize_text.py\", line 59, in field2slice\r\n    slic = slice(s1 - 1, s2)\r\nTypeError: unsupported operand type(s) for -: 'NoneType' and 'int'\r\n```\r\nThis is because of a missing None check [here](https://github.com/espnet/espnet/blob/master/espnet2/bin/tokenize_text.py#L59)\r\n\n", "before_files": [{"content": "#!/usr/bin/env python3\nimport argparse\nfrom collections import Counter\nimport logging\nfrom pathlib import Path\nimport sys\nfrom typing import List\nfrom typing import Optional\n\nfrom typeguard import check_argument_types\n\nfrom espnet.utils.cli_utils import get_commandline_args\nfrom espnet2.text.build_tokenizer import build_tokenizer\nfrom espnet2.text.cleaner import TextCleaner\nfrom espnet2.utils.types import str2bool\nfrom espnet2.utils.types import str_or_none\n\n\ndef field2slice(field: Optional[str]) -> slice:\n    \"\"\"Convert field string to slice\n\n    Note that field string accepts 1-based integer.\n\n    Examples:\n        >>> field2slice(\"1-\")\n        slice(0, None, None)\n        >>> field2slice(\"1-3\")\n        slice(0, 3, None)\n        >>> field2slice(\"-3\")\n        slice(None, 3, None)\n\n    \"\"\"\n    field = field.strip()\n    try:\n        if \"-\" in field:\n            # e.g. \"2-\" or \"2-5\" or \"-7\"\n            s1, s2 = field.split(\"-\", maxsplit=1)\n            if s1.strip() == \"\":\n                s1 = None\n            else:\n                s1 = int(s1)\n                if s1 == 0:\n                    raise ValueError(\"1-based string\")\n            if s2.strip() == \"\":\n                s2 = None\n            else:\n                s2 = int(s2)\n        else:\n            # e.g. \"2\"\n            s1 = int(field)\n            s2 = s1 + 1\n            if s1 == 0:\n                raise ValueError(\"must be 1 or more value\")\n    except ValueError:\n        raise RuntimeError(f\"Format error: e.g. '2-', '2-5', or '-5': {field}\")\n\n    # -1 because of 1-based integer following \"cut\" command\n    # e.g \"1-3\" -> slice(0, 3)\n    slic = slice(s1 - 1, s2)\n    return slic\n\n\ndef tokenize(\n    input: str,\n    output: str,\n    field: Optional[str],\n    delimiter: Optional[str],\n    token_type: str,\n    space_symbol: str,\n    non_linguistic_symbols: Optional[str],\n    bpemodel: Optional[str],\n    log_level: str,\n    write_vocabulary: bool,\n    vocabulary_size: int,\n    remove_non_linguistic_symbols: bool,\n    cutoff: int,\n    add_symbol: List[str],\n    cleaner: Optional[str],\n    g2p: Optional[str],\n):\n    assert check_argument_types()\n\n    logging.basicConfig(\n        level=log_level,\n        format=\"%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s\",\n    )\n    if input == \"-\":\n        fin = sys.stdin\n    else:\n        fin = Path(input).open(\"r\", encoding=\"utf-8\")\n    if output == \"-\":\n        fout = sys.stdout\n    else:\n        p = Path(output)\n        p.parent.mkdir(parents=True, exist_ok=True)\n        fout = p.open(\"w\", encoding=\"utf-8\")\n\n    cleaner = TextCleaner(cleaner)\n    tokenizer = build_tokenizer(\n        token_type=token_type,\n        bpemodel=bpemodel,\n        delimiter=delimiter,\n        space_symbol=space_symbol,\n        non_linguistic_symbols=non_linguistic_symbols,\n        remove_non_linguistic_symbols=remove_non_linguistic_symbols,\n        g2p_type=g2p,\n    )\n\n    counter = Counter()\n    if field is not None:\n        field = field2slice(field)\n\n    for line in fin:\n        line = line.rstrip()\n        if field is not None:\n            # e.g. field=\"2-\"\n            # uttidA hello world!! -> hello world!!\n            tokens = line.split(delimiter)\n            tokens = tokens[field]\n            if delimiter is None:\n                line = \" \".join(tokens)\n            else:\n                line = delimiter.join(tokens)\n\n        line = cleaner(line)\n        tokens = tokenizer.text2tokens(line)\n        if not write_vocabulary:\n            fout.write(\" \".join(tokens) + \"\\n\")\n        else:\n            for t in tokens:\n                counter[t] += 1\n\n    if not write_vocabulary:\n        return\n\n    # ======= write_vocabulary mode from here =======\n    # Sort by the number of occurrences in descending order\n    # and filter lower frequency words than cutoff value\n    words_and_counts = list(\n        filter(lambda x: x[1] > cutoff, sorted(counter.items(), key=lambda x: -x[1]))\n    )\n    # Restrict the vocabulary size\n    if vocabulary_size > 0:\n        if vocabulary_size < len(add_symbol):\n            raise RuntimeError(f\"vocabulary_size is too small: {vocabulary_size}\")\n        words_and_counts = words_and_counts[: vocabulary_size - len(add_symbol)]\n\n    # Parse the values of --add_symbol\n    for symbol_and_id in add_symbol:\n        # e.g symbol=\"<blank>:0\"\n        try:\n            symbol, idx = symbol_and_id.split(\":\")\n            idx = int(idx)\n        except ValueError:\n            raise RuntimeError(f\"Format error: e.g. '<blank>:0': {symbol_and_id}\")\n        symbol = symbol.strip()\n\n        # e.g. idx=0  -> append as the first symbol\n        # e.g. idx=-1 -> append as the last symbol\n        if idx < 0:\n            idx = len(words_and_counts) + 1 + idx\n        words_and_counts.insert(idx, (symbol, None))\n\n    # Write words\n    for w, c in words_and_counts:\n        fout.write(w + \"\\n\")\n\n    # Logging\n    total_count = sum(counter.values())\n    invocab_count = sum(c for w, c in words_and_counts if c is not None)\n    logging.info(f\"OOV rate = {(total_count - invocab_count) / total_count * 100} %\")\n\n\ndef get_parser() -> argparse.ArgumentParser:\n    parser = argparse.ArgumentParser(\n        description=\"Tokenize texts\",\n        formatter_class=argparse.ArgumentDefaultsHelpFormatter,\n    )\n    parser.add_argument(\n        \"--log_level\",\n        type=lambda x: x.upper(),\n        default=\"INFO\",\n        choices=(\"CRITICAL\", \"ERROR\", \"WARNING\", \"INFO\", \"DEBUG\", \"NOTSET\"),\n        help=\"The verbose level of logging\",\n    )\n\n    parser.add_argument(\n        \"--input\", \"-i\", required=True, help=\"Input text. - indicates sys.stdin\"\n    )\n    parser.add_argument(\n        \"--output\", \"-o\", required=True, help=\"Output text. - indicates sys.stdout\"\n    )\n    parser.add_argument(\n        \"--field\",\n        \"-f\",\n        help=\"The target columns of the input text as 1-based integer. e.g 2-\",\n    )\n    parser.add_argument(\n        \"--token_type\",\n        \"-t\",\n        default=\"char\",\n        choices=[\"char\", \"bpe\", \"word\", \"phn\"],\n        help=\"Token type\",\n    )\n    parser.add_argument(\"--delimiter\", \"-d\", default=None, help=\"The delimiter\")\n    parser.add_argument(\"--space_symbol\", default=\"<space>\", help=\"The space symbol\")\n    parser.add_argument(\"--bpemodel\", default=None, help=\"The bpemodel file path\")\n    parser.add_argument(\n        \"--non_linguistic_symbols\",\n        type=str_or_none,\n        help=\"non_linguistic_symbols file path\",\n    )\n    parser.add_argument(\n        \"--remove_non_linguistic_symbols\",\n        type=str2bool,\n        default=False,\n        help=\"Remove non-language-symbols from tokens\",\n    )\n    parser.add_argument(\n        \"--cleaner\",\n        type=str_or_none,\n        choices=[None, \"tacotron\", \"jaconv\", \"vietnamese\"],\n        default=None,\n        help=\"Apply text cleaning\",\n    )\n    parser.add_argument(\n        \"--g2p\",\n        type=str_or_none,\n        choices=[\n            None,\n            \"g2p_en\",\n            \"g2p_en_no_space\",\n            \"pyopenjtalk\",\n            \"pyopenjtalk_kana\",\n            \"pyopenjtalk_accent\",\n            \"pyopenjtalk_accent_with_pause\",\n            \"pypinyin_g2p\",\n            \"pypinyin_g2p_phone\",\n            \"espeak_ng_arabic\",\n        ],\n        default=None,\n        help=\"Specify g2p method if --token_type=phn\",\n    )\n\n    group = parser.add_argument_group(\"write_vocabulary mode related\")\n    group.add_argument(\n        \"--write_vocabulary\",\n        type=str2bool,\n        default=False,\n        help=\"Write tokens list instead of tokenized text per line\",\n    )\n    group.add_argument(\"--vocabulary_size\", type=int, default=0, help=\"Vocabulary size\")\n    group.add_argument(\n        \"--cutoff\",\n        default=0,\n        type=int,\n        help=\"cut-off frequency used for write-vocabulary mode\",\n    )\n    group.add_argument(\n        \"--add_symbol\",\n        type=str,\n        default=[],\n        action=\"append\",\n        help=\"Append symbol e.g. --add_symbol '<blank>:0' --add_symbol '<unk>:1'\",\n    )\n\n    return parser\n\n\ndef main(cmd=None):\n    print(get_commandline_args(), file=sys.stderr)\n    parser = get_parser()\n    args = parser.parse_args(cmd)\n    kwargs = vars(args)\n    tokenize(**kwargs)\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "espnet2/bin/tokenize_text.py"}], "after_files": [{"content": "#!/usr/bin/env python3\nimport argparse\nfrom collections import Counter\nimport logging\nfrom pathlib import Path\nimport sys\nfrom typing import List\nfrom typing import Optional\n\nfrom typeguard import check_argument_types\n\nfrom espnet.utils.cli_utils import get_commandline_args\nfrom espnet2.text.build_tokenizer import build_tokenizer\nfrom espnet2.text.cleaner import TextCleaner\nfrom espnet2.utils.types import str2bool\nfrom espnet2.utils.types import str_or_none\n\n\ndef field2slice(field: Optional[str]) -> slice:\n    \"\"\"Convert field string to slice\n\n    Note that field string accepts 1-based integer.\n\n    Examples:\n        >>> field2slice(\"1-\")\n        slice(0, None, None)\n        >>> field2slice(\"1-3\")\n        slice(0, 3, None)\n        >>> field2slice(\"-3\")\n        slice(None, 3, None)\n    \"\"\"\n    field = field.strip()\n    try:\n        if \"-\" in field:\n            # e.g. \"2-\" or \"2-5\" or \"-7\"\n            s1, s2 = field.split(\"-\", maxsplit=1)\n            if s1.strip() == \"\":\n                s1 = None\n            else:\n                s1 = int(s1)\n                if s1 == 0:\n                    raise ValueError(\"1-based string\")\n            if s2.strip() == \"\":\n                s2 = None\n            else:\n                s2 = int(s2)\n        else:\n            # e.g. \"2\"\n            s1 = int(field)\n            s2 = s1 + 1\n            if s1 == 0:\n                raise ValueError(\"must be 1 or more value\")\n    except ValueError:\n        raise RuntimeError(f\"Format error: e.g. '2-', '2-5', or '-5': {field}\")\n\n    if s1 is None:\n        slic = slice(None, s2)\n    else:\n        # -1 because of 1-based integer following \"cut\" command\n        # e.g \"1-3\" -> slice(0, 3)\n        slic = slice(s1 - 1, s2)\n    return slic\n\n\ndef tokenize(\n    input: str,\n    output: str,\n    field: Optional[str],\n    delimiter: Optional[str],\n    token_type: str,\n    space_symbol: str,\n    non_linguistic_symbols: Optional[str],\n    bpemodel: Optional[str],\n    log_level: str,\n    write_vocabulary: bool,\n    vocabulary_size: int,\n    remove_non_linguistic_symbols: bool,\n    cutoff: int,\n    add_symbol: List[str],\n    cleaner: Optional[str],\n    g2p: Optional[str],\n):\n    assert check_argument_types()\n\n    logging.basicConfig(\n        level=log_level,\n        format=\"%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s\",\n    )\n    if input == \"-\":\n        fin = sys.stdin\n    else:\n        fin = Path(input).open(\"r\", encoding=\"utf-8\")\n    if output == \"-\":\n        fout = sys.stdout\n    else:\n        p = Path(output)\n        p.parent.mkdir(parents=True, exist_ok=True)\n        fout = p.open(\"w\", encoding=\"utf-8\")\n\n    cleaner = TextCleaner(cleaner)\n    tokenizer = build_tokenizer(\n        token_type=token_type,\n        bpemodel=bpemodel,\n        delimiter=delimiter,\n        space_symbol=space_symbol,\n        non_linguistic_symbols=non_linguistic_symbols,\n        remove_non_linguistic_symbols=remove_non_linguistic_symbols,\n        g2p_type=g2p,\n    )\n\n    counter = Counter()\n    if field is not None:\n        field = field2slice(field)\n\n    for line in fin:\n        line = line.rstrip()\n        if field is not None:\n            # e.g. field=\"2-\"\n            # uttidA hello world!! -> hello world!!\n            tokens = line.split(delimiter)\n            tokens = tokens[field]\n            if delimiter is None:\n                line = \" \".join(tokens)\n            else:\n                line = delimiter.join(tokens)\n\n        line = cleaner(line)\n        tokens = tokenizer.text2tokens(line)\n        if not write_vocabulary:\n            fout.write(\" \".join(tokens) + \"\\n\")\n        else:\n            for t in tokens:\n                counter[t] += 1\n\n    if not write_vocabulary:\n        return\n\n    # ======= write_vocabulary mode from here =======\n    # Sort by the number of occurrences in descending order\n    # and filter lower frequency words than cutoff value\n    words_and_counts = list(\n        filter(lambda x: x[1] > cutoff, sorted(counter.items(), key=lambda x: -x[1]))\n    )\n    # Restrict the vocabulary size\n    if vocabulary_size > 0:\n        if vocabulary_size < len(add_symbol):\n            raise RuntimeError(f\"vocabulary_size is too small: {vocabulary_size}\")\n        words_and_counts = words_and_counts[: vocabulary_size - len(add_symbol)]\n\n    # Parse the values of --add_symbol\n    for symbol_and_id in add_symbol:\n        # e.g symbol=\"<blank>:0\"\n        try:\n            symbol, idx = symbol_and_id.split(\":\")\n            idx = int(idx)\n        except ValueError:\n            raise RuntimeError(f\"Format error: e.g. '<blank>:0': {symbol_and_id}\")\n        symbol = symbol.strip()\n\n        # e.g. idx=0  -> append as the first symbol\n        # e.g. idx=-1 -> append as the last symbol\n        if idx < 0:\n            idx = len(words_and_counts) + 1 + idx\n        words_and_counts.insert(idx, (symbol, None))\n\n    # Write words\n    for w, c in words_and_counts:\n        fout.write(w + \"\\n\")\n\n    # Logging\n    total_count = sum(counter.values())\n    invocab_count = sum(c for w, c in words_and_counts if c is not None)\n    logging.info(f\"OOV rate = {(total_count - invocab_count) / total_count * 100} %\")\n\n\ndef get_parser() -> argparse.ArgumentParser:\n    parser = argparse.ArgumentParser(\n        description=\"Tokenize texts\",\n        formatter_class=argparse.ArgumentDefaultsHelpFormatter,\n    )\n    parser.add_argument(\n        \"--log_level\",\n        type=lambda x: x.upper(),\n        default=\"INFO\",\n        choices=(\"CRITICAL\", \"ERROR\", \"WARNING\", \"INFO\", \"DEBUG\", \"NOTSET\"),\n        help=\"The verbose level of logging\",\n    )\n\n    parser.add_argument(\n        \"--input\", \"-i\", required=True, help=\"Input text. - indicates sys.stdin\"\n    )\n    parser.add_argument(\n        \"--output\", \"-o\", required=True, help=\"Output text. - indicates sys.stdout\"\n    )\n    parser.add_argument(\n        \"--field\",\n        \"-f\",\n        help=\"The target columns of the input text as 1-based integer. e.g 2-\",\n    )\n    parser.add_argument(\n        \"--token_type\",\n        \"-t\",\n        default=\"char\",\n        choices=[\"char\", \"bpe\", \"word\", \"phn\"],\n        help=\"Token type\",\n    )\n    parser.add_argument(\"--delimiter\", \"-d\", default=None, help=\"The delimiter\")\n    parser.add_argument(\"--space_symbol\", default=\"<space>\", help=\"The space symbol\")\n    parser.add_argument(\"--bpemodel\", default=None, help=\"The bpemodel file path\")\n    parser.add_argument(\n        \"--non_linguistic_symbols\",\n        type=str_or_none,\n        help=\"non_linguistic_symbols file path\",\n    )\n    parser.add_argument(\n        \"--remove_non_linguistic_symbols\",\n        type=str2bool,\n        default=False,\n        help=\"Remove non-language-symbols from tokens\",\n    )\n    parser.add_argument(\n        \"--cleaner\",\n        type=str_or_none,\n        choices=[None, \"tacotron\", \"jaconv\", \"vietnamese\"],\n        default=None,\n        help=\"Apply text cleaning\",\n    )\n    parser.add_argument(\n        \"--g2p\",\n        type=str_or_none,\n        choices=[\n            None,\n            \"g2p_en\",\n            \"g2p_en_no_space\",\n            \"pyopenjtalk\",\n            \"pyopenjtalk_kana\",\n            \"pyopenjtalk_accent\",\n            \"pyopenjtalk_accent_with_pause\",\n            \"pypinyin_g2p\",\n            \"pypinyin_g2p_phone\",\n            \"espeak_ng_arabic\",\n        ],\n        default=None,\n        help=\"Specify g2p method if --token_type=phn\",\n    )\n\n    group = parser.add_argument_group(\"write_vocabulary mode related\")\n    group.add_argument(\n        \"--write_vocabulary\",\n        type=str2bool,\n        default=False,\n        help=\"Write tokens list instead of tokenized text per line\",\n    )\n    group.add_argument(\"--vocabulary_size\", type=int, default=0, help=\"Vocabulary size\")\n    group.add_argument(\n        \"--cutoff\",\n        default=0,\n        type=int,\n        help=\"cut-off frequency used for write-vocabulary mode\",\n    )\n    group.add_argument(\n        \"--add_symbol\",\n        type=str,\n        default=[],\n        action=\"append\",\n        help=\"Append symbol e.g. --add_symbol '<blank>:0' --add_symbol '<unk>:1'\",\n    )\n\n    return parser\n\n\ndef main(cmd=None):\n    print(get_commandline_args(), file=sys.stderr)\n    parser = get_parser()\n    args = parser.parse_args(cmd)\n    kwargs = vars(args)\n    tokenize(**kwargs)\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "espnet2/bin/tokenize_text.py"}]}