We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
The scoring script is incorrect if any pruning is done on the dictionary or in the case of word models
Hi @sw005320 and other developers,

The command `json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${dir}/ref.trn --hyps ${dir}/hyp.trn` in the `score_sclite.sh` replaces all units in reference that are not in the dictionary with unk and then scores it.

This is wrong because you are modifying the reference transcription to match your dictionary. Let's say I train a model with dictionary with only 1 term "Mr" and the hypothesis is 
"Mr unk unk" and reference is "Mr John Doe". This would give a WER of 0% because the reference would be modified to "Mr unk unk".  

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `utils/json2trn.py`
Content:
```
1 #!/usr/bin/env python
2 # encoding: utf-8
3 
4 # Copyright 2017 Johns Hopkins University (Shinji Watanabe)
5 #           2018 Xuankai Chang (Shanghai Jiao Tong University)
6 #  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
7 
8 import argparse
9 import codecs
10 import json
11 import logging
12 import sys
13 
14 from espnet.utils.cli_utils import get_commandline_args
15 
16 
17 def get_parser():
18     parser = argparse.ArgumentParser(
19         description='convert a json to a transcription file with a token dictionary',
20         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
21     parser.add_argument('json', type=str, help='json files')
22     parser.add_argument('dict', type=str, help='dict')
23     parser.add_argument('--num-spkrs', type=int, default=1, help='number of speakers')
24     parser.add_argument('--refs', type=str, nargs='+', help='ref for all speakers')
25     parser.add_argument('--hyps', type=str, nargs='+', help='hyp for all outputs')
26     return parser
27 
28 
29 def main(args):
30     args = get_parser().parse_args(args)
31     convert(args.json, args.dict, args.refs, args.hyps, args.num_spkrs)
32 
33 
34 def convert(jsonf, dic, refs, hyps, num_spkrs=1):
35     n_ref = len(refs)
36     n_hyp = len(hyps)
37     assert n_ref == n_hyp
38     assert n_ref == num_spkrs
39 
40     # logging info
41     logfmt = '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s'
42     logging.basicConfig(level=logging.INFO, format=logfmt)
43     logging.info(get_commandline_args())
44 
45     logging.info("reading %s", jsonf)
46     with codecs.open(jsonf, 'r', encoding="utf-8") as f:
47         j = json.load(f)
48 
49     logging.info("reading %s", dic)
50     with codecs.open(dic, 'r', encoding="utf-8") as f:
51         dictionary = f.readlines()
52     char_list = [entry.split(' ')[0] for entry in dictionary]
53     char_list.insert(0, '<blank>')
54     char_list.append('<eos>')
55 
56     for ns in range(num_spkrs):
57         hyp_file = codecs.open(hyps[ns], 'w', encoding="utf-8")
58         ref_file = codecs.open(refs[ns], 'w', encoding="utf-8")
59 
60         for x in j['utts']:
61             # hyps
62             if num_spkrs == 1:
63                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]
64             else:
65                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]
66             hyp_file.write(" ".join(seq).replace('<eos>', '')),
67             hyp_file.write(" (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
68 
69             # ref
70             if num_spkrs == 1:
71                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['tokenid'].split()]
72             else:
73                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['tokenid'].split()]
74             ref_file.write(" ".join(seq).replace('<eos>', '')),
75             ref_file.write(" (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
76 
77         hyp_file.close()
78         ref_file.close()
79 
80 
81 if __name__ == '__main__':
82     main(sys.argv[1:])
83
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/utils/json2trn.py b/utils/json2trn.py
--- a/utils/json2trn.py
+++ b/utils/json2trn.py
@@ -58,21 +58,25 @@
         ref_file = codecs.open(refs[ns], 'w', encoding="utf-8")
 
         for x in j['utts']:
-            # hyps
+            # recognition hypothesis
             if num_spkrs == 1:
                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]
             else:
                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]
+            # In the recognition hypothesis, the <eos> symbol is usually attached in the last part of the sentence
+            # and it is removed below.
             hyp_file.write(" ".join(seq).replace('<eos>', '')),
             hyp_file.write(" (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
 
-            # ref
+            # reference
             if num_spkrs == 1:
-                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['tokenid'].split()]
+                seq = j['utts'][x]['output'][0]['token']
             else:
-                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['tokenid'].split()]
-            ref_file.write(" ".join(seq).replace('<eos>', '')),
-            ref_file.write(" (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
+                seq = j['utts'][x]['output'][ns][0]['token']
+            # Unlike the recognition hypothesis, the reference is directly generated from a token without dictionary
+            # to avoid to include <unk> symbols in the reference to make scoring normal.
+            # The detailed discussion can be found at https://github.com/espnet/espnet/issues/993
+            ref_file.write(seq + " (" + j['utts'][x]['utt2spk'].replace('-', '_') + "-" + x + ")\n")
 
         hyp_file.close()
         ref_file.close()

{"golden_diff": "diff --git a/utils/json2trn.py b/utils/json2trn.py\n--- a/utils/json2trn.py\n+++ b/utils/json2trn.py\n@@ -58,21 +58,25 @@\n         ref_file = codecs.open(refs[ns], 'w', encoding=\"utf-8\")\n \n         for x in j['utts']:\n-            # hyps\n+            # recognition hypothesis\n             if num_spkrs == 1:\n                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]\n             else:\n                 seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]\n+            # In the recognition hypothesis, the <eos> symbol is usually attached in the last part of the sentence\n+            # and it is removed below.\n             hyp_file.write(\" \".join(seq).replace('<eos>', '')),\n             hyp_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n \n-            # ref\n+            # reference\n             if num_spkrs == 1:\n-                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['tokenid'].split()]\n+                seq = j['utts'][x]['output'][0]['token']\n             else:\n-                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['tokenid'].split()]\n-            ref_file.write(\" \".join(seq).replace('<eos>', '')),\n-            ref_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n+                seq = j['utts'][x]['output'][ns][0]['token']\n+            # Unlike the recognition hypothesis, the reference is directly generated from a token without dictionary\n+            # to avoid to include <unk> symbols in the reference to make scoring normal.\n+            # The detailed discussion can be found at https://github.com/espnet/espnet/issues/993\n+            ref_file.write(seq + \" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n \n         hyp_file.close()\n         ref_file.close()\n", "issue": "The scoring script is incorrect if any pruning is done on the dictionary or in the case of word models\nHi @sw005320 and other developers,\r\n\r\nThe command `json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${dir}/ref.trn --hyps ${dir}/hyp.trn` in the `score_sclite.sh` replaces all units in reference that are not in the dictionary with unk and then scores it.\r\n\r\nThis is wrong because you are modifying the reference transcription to match your dictionary. Let's say I train a model with dictionary with only 1 term \"Mr\" and the hypothesis is \r\n\"Mr unk unk\" and reference is \"Mr John Doe\". This would give a WER of 0% because the reference would be modified to \"Mr unk unk\".  \n", "before_files": [{"content": "#!/usr/bin/env python\n# encoding: utf-8\n\n# Copyright 2017 Johns Hopkins University (Shinji Watanabe)\n#           2018 Xuankai Chang (Shanghai Jiao Tong University)\n#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)\n\nimport argparse\nimport codecs\nimport json\nimport logging\nimport sys\n\nfrom espnet.utils.cli_utils import get_commandline_args\n\n\ndef get_parser():\n    parser = argparse.ArgumentParser(\n        description='convert a json to a transcription file with a token dictionary',\n        formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n    parser.add_argument('json', type=str, help='json files')\n    parser.add_argument('dict', type=str, help='dict')\n    parser.add_argument('--num-spkrs', type=int, default=1, help='number of speakers')\n    parser.add_argument('--refs', type=str, nargs='+', help='ref for all speakers')\n    parser.add_argument('--hyps', type=str, nargs='+', help='hyp for all outputs')\n    return parser\n\n\ndef main(args):\n    args = get_parser().parse_args(args)\n    convert(args.json, args.dict, args.refs, args.hyps, args.num_spkrs)\n\n\ndef convert(jsonf, dic, refs, hyps, num_spkrs=1):\n    n_ref = len(refs)\n    n_hyp = len(hyps)\n    assert n_ref == n_hyp\n    assert n_ref == num_spkrs\n\n    # logging info\n    logfmt = '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s'\n    logging.basicConfig(level=logging.INFO, format=logfmt)\n    logging.info(get_commandline_args())\n\n    logging.info(\"reading %s\", jsonf)\n    with codecs.open(jsonf, 'r', encoding=\"utf-8\") as f:\n        j = json.load(f)\n\n    logging.info(\"reading %s\", dic)\n    with codecs.open(dic, 'r', encoding=\"utf-8\") as f:\n        dictionary = f.readlines()\n    char_list = [entry.split(' ')[0] for entry in dictionary]\n    char_list.insert(0, '<blank>')\n    char_list.append('<eos>')\n\n    for ns in range(num_spkrs):\n        hyp_file = codecs.open(hyps[ns], 'w', encoding=\"utf-8\")\n        ref_file = codecs.open(refs[ns], 'w', encoding=\"utf-8\")\n\n        for x in j['utts']:\n            # hyps\n            if num_spkrs == 1:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]\n            else:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]\n            hyp_file.write(\" \".join(seq).replace('<eos>', '')),\n            hyp_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n\n            # ref\n            if num_spkrs == 1:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['tokenid'].split()]\n            else:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['tokenid'].split()]\n            ref_file.write(\" \".join(seq).replace('<eos>', '')),\n            ref_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n\n        hyp_file.close()\n        ref_file.close()\n\n\nif __name__ == '__main__':\n    main(sys.argv[1:])\n", "path": "utils/json2trn.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# encoding: utf-8\n\n# Copyright 2017 Johns Hopkins University (Shinji Watanabe)\n#           2018 Xuankai Chang (Shanghai Jiao Tong University)\n#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)\n\nimport argparse\nimport codecs\nimport json\nimport logging\nimport sys\n\nfrom espnet.utils.cli_utils import get_commandline_args\n\n\ndef get_parser():\n    parser = argparse.ArgumentParser(\n        description='convert a json to a transcription file with a token dictionary',\n        formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n    parser.add_argument('json', type=str, help='json files')\n    parser.add_argument('dict', type=str, help='dict')\n    parser.add_argument('--num-spkrs', type=int, default=1, help='number of speakers')\n    parser.add_argument('--refs', type=str, nargs='+', help='ref for all speakers')\n    parser.add_argument('--hyps', type=str, nargs='+', help='hyp for all outputs')\n    return parser\n\n\ndef main(args):\n    args = get_parser().parse_args(args)\n    convert(args.json, args.dict, args.refs, args.hyps, args.num_spkrs)\n\n\ndef convert(jsonf, dic, refs, hyps, num_spkrs=1):\n    n_ref = len(refs)\n    n_hyp = len(hyps)\n    assert n_ref == n_hyp\n    assert n_ref == num_spkrs\n\n    # logging info\n    logfmt = '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s'\n    logging.basicConfig(level=logging.INFO, format=logfmt)\n    logging.info(get_commandline_args())\n\n    logging.info(\"reading %s\", jsonf)\n    with codecs.open(jsonf, 'r', encoding=\"utf-8\") as f:\n        j = json.load(f)\n\n    logging.info(\"reading %s\", dic)\n    with codecs.open(dic, 'r', encoding=\"utf-8\") as f:\n        dictionary = f.readlines()\n    char_list = [entry.split(' ')[0] for entry in dictionary]\n    char_list.insert(0, '<blank>')\n    char_list.append('<eos>')\n\n    for ns in range(num_spkrs):\n        hyp_file = codecs.open(hyps[ns], 'w', encoding=\"utf-8\")\n        ref_file = codecs.open(refs[ns], 'w', encoding=\"utf-8\")\n\n        for x in j['utts']:\n            # recognition hypothesis\n            if num_spkrs == 1:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][0]['rec_tokenid'].split()]\n            else:\n                seq = [char_list[int(i)] for i in j['utts'][x]['output'][ns][0]['rec_tokenid'].split()]\n            # In the recognition hypothesis, the <eos> symbol is usually attached in the last part of the sentence\n            # and it is removed below.\n            hyp_file.write(\" \".join(seq).replace('<eos>', '')),\n            hyp_file.write(\" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n\n            # reference\n            if num_spkrs == 1:\n                seq = j['utts'][x]['output'][0]['token']\n            else:\n                seq = j['utts'][x]['output'][ns][0]['token']\n            # Unlike the recognition hypothesis, the reference is directly generated from a token without dictionary\n            # to avoid to include <unk> symbols in the reference to make scoring normal.\n            # The detailed discussion can be found at https://github.com/espnet/espnet/issues/993\n            ref_file.write(seq + \" (\" + j['utts'][x]['utt2spk'].replace('-', '_') + \"-\" + x + \")\\n\")\n\n        hyp_file.close()\n        ref_file.close()\n\n\nif __name__ == '__main__':\n    main(sys.argv[1:])\n", "path": "utils/json2trn.py"}]}