We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add `lang` parameter to JavaScript CLI
Consider to add the `--lang` option to the JavaScript CLI tool similar to #55 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `budoux/main.py`
Content:
```
1 #!/usr/bin/env python
2 # Copyright 2021 Google LLC
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     https://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 """BudouX Script to provide CLI for user."""
16 import argparse
17 import glob
18 import json
19 import os
20 import shutil
21 import sys
22 import textwrap
23 import typing
24 
25 import pkg_resources
26 
27 import budoux
28 
29 ArgList = typing.Optional[typing.List[str]]
30 
31 
32 class BudouxHelpFormatter(argparse.ArgumentDefaultsHelpFormatter,
33                           argparse.RawDescriptionHelpFormatter):
34   pass
35 
36 
37 def check_file(path: str) -> str:
38   """Check if a given filepath exists or not.
39 
40   Args:
41       path (str): Model path
42 
43   Raises:
44       FileNotFoundError: Raise if given path does not exist.
45 
46   Returns:
47       str: A model path.
48   """
49   if os.path.isfile(path):
50     return path
51   else:
52     raise argparse.ArgumentTypeError(f"'{path}' is not found.")
53 
54 
55 def get_model_langs() -> typing.Dict[str, str]:
56   """Get a dictionary of model languages and its paths.
57 
58   Returns:
59       typing.Dict[str, str]: A dictionary of model languages and its paths.
60   """
61   models = glob.glob(
62       pkg_resources.resource_filename(__name__, "models") + "/*-*.json")
63   langs = {}
64   for model in models:
65     model_name = model.split(os.sep)[-1][:-5]
66     if model_name.startswith('zh-'):
67       langs[model_name] = model
68     else:
69       langs[model_name[:2]] = model
70   return langs
71 
72 
73 def check_lang(lang: str) -> str:
74   """Check if given language exists or not.
75 
76   Args:
77       lang (str): language code (e.g.: 'ja')
78 
79   Raises:
80       argparse.ArgumentTypeError: Raise if no model for given language exists.
81 
82   Returns:
83       str: A model path.
84   """
85   langs = get_model_langs()
86   if lang in langs:
87     return langs[lang]
88   else:
89     raise argparse.ArgumentTypeError(
90         f"'{lang}' does not exist in builtin models. (supported languages: {list(langs.keys())})"
91     )
92 
93 
94 def parse_args(test: ArgList = None) -> argparse.Namespace:
95   """Parse commandline arguments.
96 
97   Args:
98       test (typing.Optional[typing.List[str]], optional): Commandline args for testing. Defaults to None.
99 
100   Returns:
101       argparse.Namespace: Parsed data of args.
102   """
103   parser = argparse.ArgumentParser(
104       prog="budoux",
105       formatter_class=(lambda prog: BudouxHelpFormatter(
106           prog,
107           **{
108               "width": shutil.get_terminal_size(fallback=(120, 50)).columns,
109               "max_help_position": 30,
110           },
111       )),
112       description=textwrap.dedent("""\
113         BudouX is the successor to Budou,
114         the machine learning powered line break organizer tool."""),
115       epilog="\n- ".join(
116           ["supported languages of `-l`, `--lang`:",
117            *get_model_langs().keys()]))
118 
119   parser.add_argument("text", metavar="TXT", nargs="?", type=str, help="text")
120   parser.add_argument(
121       "-H",
122       "--html",
123       action="store_true",
124       help="HTML mode",
125   )
126   model_select_group = parser.add_mutually_exclusive_group()
127   model_select_group.add_argument(
128       "-m",
129       "--model",
130       metavar="JSON",
131       type=check_file,
132       default=pkg_resources.resource_filename(__name__, "models/ja-knbc.json"),
133       help="custom model file path",
134   )
135   model_select_group.add_argument(
136       "-l",
137       "--lang",
138       metavar="LANG",
139       type=check_lang,
140       help="language of custom model",
141   )
142   parser.add_argument(
143       "-d",
144       "--delim",
145       metavar="STR",
146       type=str,
147       default="---",
148       help="output delimiter in TEXT mode",
149   )
150   parser.add_argument(
151       "-V",
152       "--version",
153       action="version",
154       version="%(prog)s {}".format(budoux.__version__),
155   )
156   if test is not None:
157     return parser.parse_args(test)
158   else:
159     return parser.parse_args()
160 
161 
162 def _main(test: ArgList = None) -> str:
163   args = parse_args(test=test)
164   model_path = args.lang or args.model
165   with open(model_path, 'r', encoding='utf-8') as f:
166     model = json.load(f)
167 
168   parser = budoux.Parser(model)
169   if args.html:
170     if args.text is None:
171       inputs_html = sys.stdin.read()
172     else:
173       inputs_html = args.text
174     res = parser.translate_html_string(inputs_html)
175   else:
176     if args.text is None:
177       inputs = [v.rstrip() for v in sys.stdin.readlines()]
178     else:
179       inputs = [v.rstrip() for v in args.text.splitlines()]
180     outputs = [parser.parse(sentence) for sentence in inputs]
181     conbined_output = ["\n".join(output) for output in outputs]
182     ors = "\n" + args.delim + "\n"
183     res = ors.join(conbined_output)
184 
185   return res
186 
187 
188 def main(test: ArgList = None) -> None:
189   try:
190     print(_main(test))
191   except KeyboardInterrupt:
192     exit(0)
193 
194 
195 if __name__ == "__main__":
196   main()
197
```

Path: `budoux/parser.py`
Content:
```
1 # Copyright 2021 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     https://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """BudouX parser to provide semantic chunks."""
15 
16 import json
17 import os
18 import typing
19 from html.parser import HTMLParser
20 
21 from .feature_extractor import get_feature
22 from .utils import INVALID, SEP
23 
24 MODEL_DIR = os.path.join(os.path.dirname(__file__), 'models')
25 PARENT_CSS_STYLE = 'word-break: keep-all; overflow-wrap: break-word;'
26 with open(
27     os.path.join(os.path.dirname(__file__), 'skip_nodes.json'),
28     encoding='utf-8') as f:
29   SKIP_NODES: typing.Set[str] = set(json.load(f))
30 
31 HTMLAttr = typing.List[typing.Tuple[str, typing.Union[str, None]]]
32 
33 
34 class TextContentExtractor(HTMLParser):
35   """An HTML parser to extract text content.
36 
37   Attributes:
38     output (str): Accumulated text content.
39   """
40   output = ''
41 
42   def handle_data(self, data: str) -> None:
43     self.output += data
44 
45 
46 class HTMLChunkResolver(HTMLParser):
47   """An HTML parser to resolve the given HTML string and semantic chunks.
48 
49   Attributes:
50     output (str): The HTML string to output.
51     chunks (str): Chunks concatenated with the separator string.
52   """
53   output = ''
54 
55   def __init__(self, chunks: typing.List[str]):
56     """Initializes the parser.
57 
58     Args:
59       chunks (List[str]): The chunks to resolve.
60     """
61     HTMLParser.__init__(self)
62     self.chunks_joined = SEP.join(chunks)
63     self.to_skip = False
64 
65   def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:
66     attr_pairs = []
67     for attr in attrs:
68       if attr[1] is None:
69         attr_pairs.append(attr[0])
70       else:
71         attr_pairs.append('%s="%s"' % (attr[0], attr[1]))
72     encoded_attrs = ' '.join(attr_pairs)
73     self.output += '<%s %s>' % (tag, encoded_attrs)
74     self.to_skip = tag.upper() in SKIP_NODES
75 
76   def handle_endtag(self, tag: str) -> None:
77     self.output += '</%s>' % (tag)
78     self.to_skip = False
79 
80   def handle_data(self, data: str) -> None:
81     if self.to_skip:
82       self.output += data
83       if self.chunks_joined[0] == SEP:
84         self.chunks_joined = self.chunks_joined[1 + len(data):]
85       else:
86         self.chunks_joined = self.chunks_joined[len(data):]
87       return None
88     for char in data:
89       if char == self.chunks_joined[0]:
90         self.chunks_joined = self.chunks_joined[1:]
91         self.output += char
92       else:
93         self.chunks_joined = self.chunks_joined[2:]
94         self.output += '<wbr>' + char
95 
96 
97 class Parser:
98   """BudouX's Parser.
99 
100   The main parser object with a variety of class methods to provide semantic
101   chunks and markups from the given input string.
102 
103   Attributes:
104     model: A dict mapping a feature (str) and its score (int).
105   """
106 
107   def __init__(self, model: typing.Dict[str, int]):
108     """Initializes the parser.
109 
110     Args:
111       model (Dict[str, int]): A dict mapping a feature and its score.
112     """
113     self.model = model
114 
115   def parse(self, sentence: str) -> typing.List[str]:
116     """Parses the input sentence and returns a list of semantic chunks.
117 
118     Args:
119       sentence (str): An input sentence.
120 
121     Returns:
122       A list of semantic chunks (List[str]).
123     """
124     if sentence == '':
125       return []
126     chunks = [sentence[0]]
127     base_score = -sum(self.model.values())
128     for i in range(1, len(sentence)):
129       feature = get_feature(
130           sentence[i - 3] if i > 2 else INVALID,
131           sentence[i - 2] if i > 1 else INVALID, sentence[i - 1], sentence[i],
132           sentence[i + 1] if i + 1 < len(sentence) else INVALID,
133           sentence[i + 2] if i + 2 < len(sentence) else INVALID)
134       score = base_score + 2 * sum(
135           self.model[f] for f in feature if f in self.model)
136       if score > 0:
137         chunks.append(sentence[i])
138       else:
139         chunks[-1] += sentence[i]
140     return chunks
141 
142   def translate_html_string(self, html: str) -> str:
143     """Translates the given HTML string with markups for semantic line breaks.
144 
145     Args:
146       html (str): An input html string.
147 
148     Returns:
149       The translated HTML string (str).
150     """
151     # TODO: Align with the JavaScript API regarding the parent element addition.
152     text_content_extractor = TextContentExtractor()
153     text_content_extractor.feed(html)
154     text_content = text_content_extractor.output
155     chunks = self.parse(text_content)
156     resolver = HTMLChunkResolver(chunks)
157     resolver.feed(html)
158     return '<span style="%s">%s</span>' % (PARENT_CSS_STYLE, resolver.output)
159 
160 
161 def load_default_japanese_parser() -> Parser:
162   """Loads a parser equipped with the default Japanese model.
163 
164   Returns:
165     A parser (:obj:`budoux.Parser`).
166   """
167   with open(os.path.join(MODEL_DIR, 'ja-knbc.json'), encoding='utf-8') as f:
168     model = json.load(f)
169   return Parser(model)
170 
171 
172 def load_default_simplified_chinese_parser() -> Parser:
173   """Loads a parser equipped with the default Simplified Chinese model.
174 
175   Returns:
176     A parser (:obj:`budoux.Parser`).
177   """
178   with open(os.path.join(MODEL_DIR, 'zh-hans.json'), encoding='utf-8') as f:
179     model = json.load(f)
180   return Parser(model)
181 
182 
183 def load_default_traditional_chinese_parser() -> Parser:
184   """Loads a parser equipped with the default Traditional Chinese model.
185 
186   Returns:
187     A parser (:obj:`budoux.Parser`).
188   """
189   with open(os.path.join(MODEL_DIR, 'zh-hant.json'), encoding='utf-8') as f:
190     model = json.load(f)
191   return Parser(model)
192
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/budoux/main.py b/budoux/main.py
--- a/budoux/main.py
+++ b/budoux/main.py
@@ -59,14 +59,11 @@
       typing.Dict[str, str]: A dictionary of model languages and its paths.
   """
   models = glob.glob(
-      pkg_resources.resource_filename(__name__, "models") + "/*-*.json")
+      pkg_resources.resource_filename(__name__, "models") + "/*.json")
   langs = {}
   for model in models:
-    model_name = model.split(os.sep)[-1][:-5]
-    if model_name.startswith('zh-'):
-      langs[model_name] = model
-    else:
-      langs[model_name[:2]] = model
+    lang = model.split(os.sep)[-1][:-5]
+    langs[lang] = model
   return langs
 
 
@@ -129,7 +126,7 @@
       "--model",
       metavar="JSON",
       type=check_file,
-      default=pkg_resources.resource_filename(__name__, "models/ja-knbc.json"),
+      default=pkg_resources.resource_filename(__name__, "models/ja.json"),
       help="custom model file path",
   )
   model_select_group.add_argument(
diff --git a/budoux/parser.py b/budoux/parser.py
--- a/budoux/parser.py
+++ b/budoux/parser.py
@@ -164,7 +164,7 @@
   Returns:
     A parser (:obj:`budoux.Parser`).
   """
-  with open(os.path.join(MODEL_DIR, 'ja-knbc.json'), encoding='utf-8') as f:
+  with open(os.path.join(MODEL_DIR, 'ja.json'), encoding='utf-8') as f:
     model = json.load(f)
   return Parser(model)

{"golden_diff": "diff --git a/budoux/main.py b/budoux/main.py\n--- a/budoux/main.py\n+++ b/budoux/main.py\n@@ -59,14 +59,11 @@\n       typing.Dict[str, str]: A dictionary of model languages and its paths.\n   \"\"\"\n   models = glob.glob(\n-      pkg_resources.resource_filename(__name__, \"models\") + \"/*-*.json\")\n+      pkg_resources.resource_filename(__name__, \"models\") + \"/*.json\")\n   langs = {}\n   for model in models:\n-    model_name = model.split(os.sep)[-1][:-5]\n-    if model_name.startswith('zh-'):\n-      langs[model_name] = model\n-    else:\n-      langs[model_name[:2]] = model\n+    lang = model.split(os.sep)[-1][:-5]\n+    langs[lang] = model\n   return langs\n \n \n@@ -129,7 +126,7 @@\n       \"--model\",\n       metavar=\"JSON\",\n       type=check_file,\n-      default=pkg_resources.resource_filename(__name__, \"models/ja-knbc.json\"),\n+      default=pkg_resources.resource_filename(__name__, \"models/ja.json\"),\n       help=\"custom model file path\",\n   )\n   model_select_group.add_argument(\ndiff --git a/budoux/parser.py b/budoux/parser.py\n--- a/budoux/parser.py\n+++ b/budoux/parser.py\n@@ -164,7 +164,7 @@\n   Returns:\n     A parser (:obj:`budoux.Parser`).\n   \"\"\"\n-  with open(os.path.join(MODEL_DIR, 'ja-knbc.json'), encoding='utf-8') as f:\n+  with open(os.path.join(MODEL_DIR, 'ja.json'), encoding='utf-8') as f:\n     model = json.load(f)\n   return Parser(model)\n", "issue": "Add `lang` parameter to JavaScript CLI\nConsider to add the `--lang` option to the JavaScript CLI tool similar to #55 \n", "before_files": [{"content": "#!/usr/bin/env python\n# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"BudouX Script to provide CLI for user.\"\"\"\nimport argparse\nimport glob\nimport json\nimport os\nimport shutil\nimport sys\nimport textwrap\nimport typing\n\nimport pkg_resources\n\nimport budoux\n\nArgList = typing.Optional[typing.List[str]]\n\n\nclass BudouxHelpFormatter(argparse.ArgumentDefaultsHelpFormatter,\n                          argparse.RawDescriptionHelpFormatter):\n  pass\n\n\ndef check_file(path: str) -> str:\n  \"\"\"Check if a given filepath exists or not.\n\n  Args:\n      path (str): Model path\n\n  Raises:\n      FileNotFoundError: Raise if given path does not exist.\n\n  Returns:\n      str: A model path.\n  \"\"\"\n  if os.path.isfile(path):\n    return path\n  else:\n    raise argparse.ArgumentTypeError(f\"'{path}' is not found.\")\n\n\ndef get_model_langs() -> typing.Dict[str, str]:\n  \"\"\"Get a dictionary of model languages and its paths.\n\n  Returns:\n      typing.Dict[str, str]: A dictionary of model languages and its paths.\n  \"\"\"\n  models = glob.glob(\n      pkg_resources.resource_filename(__name__, \"models\") + \"/*-*.json\")\n  langs = {}\n  for model in models:\n    model_name = model.split(os.sep)[-1][:-5]\n    if model_name.startswith('zh-'):\n      langs[model_name] = model\n    else:\n      langs[model_name[:2]] = model\n  return langs\n\n\ndef check_lang(lang: str) -> str:\n  \"\"\"Check if given language exists or not.\n\n  Args:\n      lang (str): language code (e.g.: 'ja')\n\n  Raises:\n      argparse.ArgumentTypeError: Raise if no model for given language exists.\n\n  Returns:\n      str: A model path.\n  \"\"\"\n  langs = get_model_langs()\n  if lang in langs:\n    return langs[lang]\n  else:\n    raise argparse.ArgumentTypeError(\n        f\"'{lang}' does not exist in builtin models. (supported languages: {list(langs.keys())})\"\n    )\n\n\ndef parse_args(test: ArgList = None) -> argparse.Namespace:\n  \"\"\"Parse commandline arguments.\n\n  Args:\n      test (typing.Optional[typing.List[str]], optional): Commandline args for testing. Defaults to None.\n\n  Returns:\n      argparse.Namespace: Parsed data of args.\n  \"\"\"\n  parser = argparse.ArgumentParser(\n      prog=\"budoux\",\n      formatter_class=(lambda prog: BudouxHelpFormatter(\n          prog,\n          **{\n              \"width\": shutil.get_terminal_size(fallback=(120, 50)).columns,\n              \"max_help_position\": 30,\n          },\n      )),\n      description=textwrap.dedent(\"\"\"\\\n        BudouX is the successor to Budou,\n        the machine learning powered line break organizer tool.\"\"\"),\n      epilog=\"\\n- \".join(\n          [\"supported languages of `-l`, `--lang`:\",\n           *get_model_langs().keys()]))\n\n  parser.add_argument(\"text\", metavar=\"TXT\", nargs=\"?\", type=str, help=\"text\")\n  parser.add_argument(\n      \"-H\",\n      \"--html\",\n      action=\"store_true\",\n      help=\"HTML mode\",\n  )\n  model_select_group = parser.add_mutually_exclusive_group()\n  model_select_group.add_argument(\n      \"-m\",\n      \"--model\",\n      metavar=\"JSON\",\n      type=check_file,\n      default=pkg_resources.resource_filename(__name__, \"models/ja-knbc.json\"),\n      help=\"custom model file path\",\n  )\n  model_select_group.add_argument(\n      \"-l\",\n      \"--lang\",\n      metavar=\"LANG\",\n      type=check_lang,\n      help=\"language of custom model\",\n  )\n  parser.add_argument(\n      \"-d\",\n      \"--delim\",\n      metavar=\"STR\",\n      type=str,\n      default=\"---\",\n      help=\"output delimiter in TEXT mode\",\n  )\n  parser.add_argument(\n      \"-V\",\n      \"--version\",\n      action=\"version\",\n      version=\"%(prog)s {}\".format(budoux.__version__),\n  )\n  if test is not None:\n    return parser.parse_args(test)\n  else:\n    return parser.parse_args()\n\n\ndef _main(test: ArgList = None) -> str:\n  args = parse_args(test=test)\n  model_path = args.lang or args.model\n  with open(model_path, 'r', encoding='utf-8') as f:\n    model = json.load(f)\n\n  parser = budoux.Parser(model)\n  if args.html:\n    if args.text is None:\n      inputs_html = sys.stdin.read()\n    else:\n      inputs_html = args.text\n    res = parser.translate_html_string(inputs_html)\n  else:\n    if args.text is None:\n      inputs = [v.rstrip() for v in sys.stdin.readlines()]\n    else:\n      inputs = [v.rstrip() for v in args.text.splitlines()]\n    outputs = [parser.parse(sentence) for sentence in inputs]\n    conbined_output = [\"\\n\".join(output) for output in outputs]\n    ors = \"\\n\" + args.delim + \"\\n\"\n    res = ors.join(conbined_output)\n\n  return res\n\n\ndef main(test: ArgList = None) -> None:\n  try:\n    print(_main(test))\n  except KeyboardInterrupt:\n    exit(0)\n\n\nif __name__ == \"__main__\":\n  main()\n", "path": "budoux/main.py"}, {"content": "# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"BudouX parser to provide semantic chunks.\"\"\"\n\nimport json\nimport os\nimport typing\nfrom html.parser import HTMLParser\n\nfrom .feature_extractor import get_feature\nfrom .utils import INVALID, SEP\n\nMODEL_DIR = os.path.join(os.path.dirname(__file__), 'models')\nPARENT_CSS_STYLE = 'word-break: keep-all; overflow-wrap: break-word;'\nwith open(\n    os.path.join(os.path.dirname(__file__), 'skip_nodes.json'),\n    encoding='utf-8') as f:\n  SKIP_NODES: typing.Set[str] = set(json.load(f))\n\nHTMLAttr = typing.List[typing.Tuple[str, typing.Union[str, None]]]\n\n\nclass TextContentExtractor(HTMLParser):\n  \"\"\"An HTML parser to extract text content.\n\n  Attributes:\n    output (str): Accumulated text content.\n  \"\"\"\n  output = ''\n\n  def handle_data(self, data: str) -> None:\n    self.output += data\n\n\nclass HTMLChunkResolver(HTMLParser):\n  \"\"\"An HTML parser to resolve the given HTML string and semantic chunks.\n\n  Attributes:\n    output (str): The HTML string to output.\n    chunks (str): Chunks concatenated with the separator string.\n  \"\"\"\n  output = ''\n\n  def __init__(self, chunks: typing.List[str]):\n    \"\"\"Initializes the parser.\n\n    Args:\n      chunks (List[str]): The chunks to resolve.\n    \"\"\"\n    HTMLParser.__init__(self)\n    self.chunks_joined = SEP.join(chunks)\n    self.to_skip = False\n\n  def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:\n    attr_pairs = []\n    for attr in attrs:\n      if attr[1] is None:\n        attr_pairs.append(attr[0])\n      else:\n        attr_pairs.append('%s=\"%s\"' % (attr[0], attr[1]))\n    encoded_attrs = ' '.join(attr_pairs)\n    self.output += '<%s %s>' % (tag, encoded_attrs)\n    self.to_skip = tag.upper() in SKIP_NODES\n\n  def handle_endtag(self, tag: str) -> None:\n    self.output += '</%s>' % (tag)\n    self.to_skip = False\n\n  def handle_data(self, data: str) -> None:\n    if self.to_skip:\n      self.output += data\n      if self.chunks_joined[0] == SEP:\n        self.chunks_joined = self.chunks_joined[1 + len(data):]\n      else:\n        self.chunks_joined = self.chunks_joined[len(data):]\n      return None\n    for char in data:\n      if char == self.chunks_joined[0]:\n        self.chunks_joined = self.chunks_joined[1:]\n        self.output += char\n      else:\n        self.chunks_joined = self.chunks_joined[2:]\n        self.output += '<wbr>' + char\n\n\nclass Parser:\n  \"\"\"BudouX's Parser.\n\n  The main parser object with a variety of class methods to provide semantic\n  chunks and markups from the given input string.\n\n  Attributes:\n    model: A dict mapping a feature (str) and its score (int).\n  \"\"\"\n\n  def __init__(self, model: typing.Dict[str, int]):\n    \"\"\"Initializes the parser.\n\n    Args:\n      model (Dict[str, int]): A dict mapping a feature and its score.\n    \"\"\"\n    self.model = model\n\n  def parse(self, sentence: str) -> typing.List[str]:\n    \"\"\"Parses the input sentence and returns a list of semantic chunks.\n\n    Args:\n      sentence (str): An input sentence.\n\n    Returns:\n      A list of semantic chunks (List[str]).\n    \"\"\"\n    if sentence == '':\n      return []\n    chunks = [sentence[0]]\n    base_score = -sum(self.model.values())\n    for i in range(1, len(sentence)):\n      feature = get_feature(\n          sentence[i - 3] if i > 2 else INVALID,\n          sentence[i - 2] if i > 1 else INVALID, sentence[i - 1], sentence[i],\n          sentence[i + 1] if i + 1 < len(sentence) else INVALID,\n          sentence[i + 2] if i + 2 < len(sentence) else INVALID)\n      score = base_score + 2 * sum(\n          self.model[f] for f in feature if f in self.model)\n      if score > 0:\n        chunks.append(sentence[i])\n      else:\n        chunks[-1] += sentence[i]\n    return chunks\n\n  def translate_html_string(self, html: str) -> str:\n    \"\"\"Translates the given HTML string with markups for semantic line breaks.\n\n    Args:\n      html (str): An input html string.\n\n    Returns:\n      The translated HTML string (str).\n    \"\"\"\n    # TODO: Align with the JavaScript API regarding the parent element addition.\n    text_content_extractor = TextContentExtractor()\n    text_content_extractor.feed(html)\n    text_content = text_content_extractor.output\n    chunks = self.parse(text_content)\n    resolver = HTMLChunkResolver(chunks)\n    resolver.feed(html)\n    return '<span style=\"%s\">%s</span>' % (PARENT_CSS_STYLE, resolver.output)\n\n\ndef load_default_japanese_parser() -> Parser:\n  \"\"\"Loads a parser equipped with the default Japanese model.\n\n  Returns:\n    A parser (:obj:`budoux.Parser`).\n  \"\"\"\n  with open(os.path.join(MODEL_DIR, 'ja-knbc.json'), encoding='utf-8') as f:\n    model = json.load(f)\n  return Parser(model)\n\n\ndef load_default_simplified_chinese_parser() -> Parser:\n  \"\"\"Loads a parser equipped with the default Simplified Chinese model.\n\n  Returns:\n    A parser (:obj:`budoux.Parser`).\n  \"\"\"\n  with open(os.path.join(MODEL_DIR, 'zh-hans.json'), encoding='utf-8') as f:\n    model = json.load(f)\n  return Parser(model)\n\n\ndef load_default_traditional_chinese_parser() -> Parser:\n  \"\"\"Loads a parser equipped with the default Traditional Chinese model.\n\n  Returns:\n    A parser (:obj:`budoux.Parser`).\n  \"\"\"\n  with open(os.path.join(MODEL_DIR, 'zh-hant.json'), encoding='utf-8') as f:\n    model = json.load(f)\n  return Parser(model)\n", "path": "budoux/parser.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"BudouX Script to provide CLI for user.\"\"\"\nimport argparse\nimport glob\nimport json\nimport os\nimport shutil\nimport sys\nimport textwrap\nimport typing\n\nimport pkg_resources\n\nimport budoux\n\nArgList = typing.Optional[typing.List[str]]\n\n\nclass BudouxHelpFormatter(argparse.ArgumentDefaultsHelpFormatter,\n                          argparse.RawDescriptionHelpFormatter):\n  pass\n\n\ndef check_file(path: str) -> str:\n  \"\"\"Check if a given filepath exists or not.\n\n  Args:\n      path (str): Model path\n\n  Raises:\n      FileNotFoundError: Raise if given path does not exist.\n\n  Returns:\n      str: A model path.\n  \"\"\"\n  if os.path.isfile(path):\n    return path\n  else:\n    raise argparse.ArgumentTypeError(f\"'{path}' is not found.\")\n\n\ndef get_model_langs() -> typing.Dict[str, str]:\n  \"\"\"Get a dictionary of model languages and its paths.\n\n  Returns:\n      typing.Dict[str, str]: A dictionary of model languages and its paths.\n  \"\"\"\n  models = glob.glob(\n      pkg_resources.resource_filename(__name__, \"models\") + \"/*.json\")\n  langs = {}\n  for model in models:\n    lang = model.split(os.sep)[-1][:-5]\n    langs[lang] = model\n  return langs\n\n\ndef check_lang(lang: str) -> str:\n  \"\"\"Check if given language exists or not.\n\n  Args:\n      lang (str): language code (e.g.: 'ja')\n\n  Raises:\n      argparse.ArgumentTypeError: Raise if no model for given language exists.\n\n  Returns:\n      str: A model path.\n  \"\"\"\n  langs = get_model_langs()\n  if lang in langs:\n    return langs[lang]\n  else:\n    raise argparse.ArgumentTypeError(\n        f\"'{lang}' does not exist in builtin models. (supported languages: {list(langs.keys())})\"\n    )\n\n\ndef parse_args(test: ArgList = None) -> argparse.Namespace:\n  \"\"\"Parse commandline arguments.\n\n  Args:\n      test (typing.Optional[typing.List[str]], optional): Commandline args for testing. Defaults to None.\n\n  Returns:\n      argparse.Namespace: Parsed data of args.\n  \"\"\"\n  parser = argparse.ArgumentParser(\n      prog=\"budoux\",\n      formatter_class=(lambda prog: BudouxHelpFormatter(\n          prog,\n          **{\n              \"width\": shutil.get_terminal_size(fallback=(120, 50)).columns,\n              \"max_help_position\": 30,\n          },\n      )),\n      description=textwrap.dedent(\"\"\"\\\n        BudouX is the successor to Budou,\n        the machine learning powered line break organizer tool.\"\"\"),\n      epilog=\"\\n- \".join(\n          [\"supported languages of `-l`, `--lang`:\",\n           *get_model_langs().keys()]))\n\n  parser.add_argument(\"text\", metavar=\"TXT\", nargs=\"?\", type=str, help=\"text\")\n  parser.add_argument(\n      \"-H\",\n      \"--html\",\n      action=\"store_true\",\n      help=\"HTML mode\",\n  )\n  model_select_group = parser.add_mutually_exclusive_group()\n  model_select_group.add_argument(\n      \"-m\",\n      \"--model\",\n      metavar=\"JSON\",\n      type=check_file,\n      default=pkg_resources.resource_filename(__name__, \"models/ja.json\"),\n      help=\"custom model file path\",\n  )\n  model_select_group.add_argument(\n      \"-l\",\n      \"--lang\",\n      metavar=\"LANG\",\n      type=check_lang,\n      help=\"language of custom model\",\n  )\n  parser.add_argument(\n      \"-d\",\n      \"--delim\",\n      metavar=\"STR\",\n      type=str,\n      default=\"---\",\n      help=\"output delimiter in TEXT mode\",\n  )\n  parser.add_argument(\n      \"-V\",\n      \"--version\",\n      action=\"version\",\n      version=\"%(prog)s {}\".format(budoux.__version__),\n  )\n  if test is not None:\n    return parser.parse_args(test)\n  else:\n    return parser.parse_args()\n\n\ndef _main(test: ArgList = None) -> str:\n  args = parse_args(test=test)\n  model_path = args.lang or args.model\n  with open(model_path, 'r', encoding='utf-8') as f:\n    model = json.load(f)\n\n  parser = budoux.Parser(model)\n  if args.html:\n    if args.text is None:\n      inputs_html = sys.stdin.read()\n    else:\n      inputs_html = args.text\n    res = parser.translate_html_string(inputs_html)\n  else:\n    if args.text is None:\n      inputs = [v.rstrip() for v in sys.stdin.readlines()]\n    else:\n      inputs = [v.rstrip() for v in args.text.splitlines()]\n    outputs = [parser.parse(sentence) for sentence in inputs]\n    conbined_output = [\"\\n\".join(output) for output in outputs]\n    ors = \"\\n\" + args.delim + \"\\n\"\n    res = ors.join(conbined_output)\n\n  return res\n\n\ndef main(test: ArgList = None) -> None:\n  try:\n    print(_main(test))\n  except KeyboardInterrupt:\n    exit(0)\n\n\nif __name__ == \"__main__\":\n  main()\n", "path": "budoux/main.py"}, {"content": "# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"BudouX parser to provide semantic chunks.\"\"\"\n\nimport json\nimport os\nimport typing\nfrom html.parser import HTMLParser\n\nfrom .feature_extractor import get_feature\nfrom .utils import INVALID, SEP\n\nMODEL_DIR = os.path.join(os.path.dirname(__file__), 'models')\nPARENT_CSS_STYLE = 'word-break: keep-all; overflow-wrap: break-word;'\nwith open(\n    os.path.join(os.path.dirname(__file__), 'skip_nodes.json'),\n    encoding='utf-8') as f:\n  SKIP_NODES: typing.Set[str] = set(json.load(f))\n\nHTMLAttr = typing.List[typing.Tuple[str, typing.Union[str, None]]]\n\n\nclass TextContentExtractor(HTMLParser):\n  \"\"\"An HTML parser to extract text content.\n\n  Attributes:\n    output (str): Accumulated text content.\n  \"\"\"\n  output = ''\n\n  def handle_data(self, data: str) -> None:\n    self.output += data\n\n\nclass HTMLChunkResolver(HTMLParser):\n  \"\"\"An HTML parser to resolve the given HTML string and semantic chunks.\n\n  Attributes:\n    output (str): The HTML string to output.\n    chunks (str): Chunks concatenated with the separator string.\n  \"\"\"\n  output = ''\n\n  def __init__(self, chunks: typing.List[str]):\n    \"\"\"Initializes the parser.\n\n    Args:\n      chunks (List[str]): The chunks to resolve.\n    \"\"\"\n    HTMLParser.__init__(self)\n    self.chunks_joined = SEP.join(chunks)\n    self.to_skip = False\n\n  def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:\n    attr_pairs = []\n    for attr in attrs:\n      if attr[1] is None:\n        attr_pairs.append(attr[0])\n      else:\n        attr_pairs.append('%s=\"%s\"' % (attr[0], attr[1]))\n    encoded_attrs = ' '.join(attr_pairs)\n    self.output += '<%s %s>' % (tag, encoded_attrs)\n    self.to_skip = tag.upper() in SKIP_NODES\n\n  def handle_endtag(self, tag: str) -> None:\n    self.output += '</%s>' % (tag)\n    self.to_skip = False\n\n  def handle_data(self, data: str) -> None:\n    if self.to_skip:\n      self.output += data\n      if self.chunks_joined[0] == SEP:\n        self.chunks_joined = self.chunks_joined[1 + len(data):]\n      else:\n        self.chunks_joined = self.chunks_joined[len(data):]\n      return None\n    for char in data:\n      if char == self.chunks_joined[0]:\n        self.chunks_joined = self.chunks_joined[1:]\n        self.output += char\n      else:\n        self.chunks_joined = self.chunks_joined[2:]\n        self.output += '<wbr>' + char\n\n\nclass Parser:\n  \"\"\"BudouX's Parser.\n\n  The main parser object with a variety of class methods to provide semantic\n  chunks and markups from the given input string.\n\n  Attributes:\n    model: A dict mapping a feature (str) and its score (int).\n  \"\"\"\n\n  def __init__(self, model: typing.Dict[str, int]):\n    \"\"\"Initializes the parser.\n\n    Args:\n      model (Dict[str, int]): A dict mapping a feature and its score.\n    \"\"\"\n    self.model = model\n\n  def parse(self, sentence: str) -> typing.List[str]:\n    \"\"\"Parses the input sentence and returns a list of semantic chunks.\n\n    Args:\n      sentence (str): An input sentence.\n\n    Returns:\n      A list of semantic chunks (List[str]).\n    \"\"\"\n    if sentence == '':\n      return []\n    chunks = [sentence[0]]\n    base_score = -sum(self.model.values())\n    for i in range(1, len(sentence)):\n      feature = get_feature(\n          sentence[i - 3] if i > 2 else INVALID,\n          sentence[i - 2] if i > 1 else INVALID, sentence[i - 1], sentence[i],\n          sentence[i + 1] if i + 1 < len(sentence) else INVALID,\n          sentence[i + 2] if i + 2 < len(sentence) else INVALID)\n      score = base_score + 2 * sum(\n          self.model[f] for f in feature if f in self.model)\n      if score > 0:\n        chunks.append(sentence[i])\n      else:\n        chunks[-1] += sentence[i]\n    return chunks\n\n  def translate_html_string(self, html: str) -> str:\n    \"\"\"Translates the given HTML string with markups for semantic line breaks.\n\n    Args:\n      html (str): An input html string.\n\n    Returns:\n      The translated HTML string (str).\n    \"\"\"\n    # TODO: Align with the JavaScript API regarding the parent element addition.\n    text_content_extractor = TextContentExtractor()\n    text_content_extractor.feed(html)\n    text_content = text_content_extractor.output\n    chunks = self.parse(text_content)\n    resolver = HTMLChunkResolver(chunks)\n    resolver.feed(html)\n    return '<span style=\"%s\">%s</span>' % (PARENT_CSS_STYLE, resolver.output)\n\n\ndef load_default_japanese_parser() -> Parser:\n  \"\"\"Loads a parser equipped with the default Japanese model.\n\n  Returns:\n    A parser (:obj:`budoux.Parser`).\n  \"\"\"\n  with open(os.path.join(MODEL_DIR, 'ja.json'), encoding='utf-8') as f:\n    model = json.load(f)\n  return Parser(model)\n\n\ndef load_default_simplified_chinese_parser() -> Parser:\n  \"\"\"Loads a parser equipped with the default Simplified Chinese model.\n\n  Returns:\n    A parser (:obj:`budoux.Parser`).\n  \"\"\"\n  with open(os.path.join(MODEL_DIR, 'zh-hans.json'), encoding='utf-8') as f:\n    model = json.load(f)\n  return Parser(model)\n\n\ndef load_default_traditional_chinese_parser() -> Parser:\n  \"\"\"Loads a parser equipped with the default Traditional Chinese model.\n\n  Returns:\n    A parser (:obj:`budoux.Parser`).\n  \"\"\"\n  with open(os.path.join(MODEL_DIR, 'zh-hant.json'), encoding='utf-8') as f:\n    model = json.load(f)\n  return Parser(model)\n", "path": "budoux/parser.py"}]}