We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
bad syntax in preprocess.py
levinth@zt-gpu-lin-1:~/fairseq-py$ python3 preprocess.py --source-lang de --target-lang en   --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test   --destdir data-bin/iwslt14.tokenized.de-en
  File "preprocess.py", line 84
    output_text_file = os.path.join(args.destdir, f'{output_prefix}.{lang}')
                                                                          ^
SyntaxError: invalid syntax

replacing line 84 with
output_text_file = os.path.join(args.destdir, '{output_prefix}.{lang}')
appears to have fixed the issue

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `preprocess.py`
Content:
```
1 #!/usr/bin/env python3
2 # Copyright (c) 2017-present, Facebook, Inc.
3 # All rights reserved.
4 #
5 # This source code is licensed under the license found in the LICENSE file in
6 # the root directory of this source tree. An additional grant of patent rights
7 # can be found in the PATENTS file in the same directory.
8 #
9 
10 import argparse
11 from itertools import zip_longest
12 import os
13 import shutil
14 
15 from fairseq import dictionary, indexed_dataset
16 from fairseq.tokenizer import Tokenizer
17 
18 
19 def main():
20     parser = argparse.ArgumentParser(
21         description='Data pre-processing: Create dictionary and store data in binary format')
22     parser.add_argument('-s', '--source-lang', default=None, metavar='SRC', help='source language')
23     parser.add_argument('-t', '--target-lang', default=None, metavar='TARGET', help='target language')
24     parser.add_argument('--trainpref', metavar='FP', default='train', help='target language')
25     parser.add_argument('--validpref', metavar='FP', default='valid', help='comma separated, valid language prefixes')
26     parser.add_argument('--testpref', metavar='FP', default='test', help='comma separated, test language prefixes')
27     parser.add_argument('--destdir', metavar='DIR', default='data-bin', help='destination dir')
28     parser.add_argument('--thresholdtgt', metavar='N', default=0, type=int,
29                         help='map words appearing less than threshold times to unknown')
30     parser.add_argument('--thresholdsrc', metavar='N', default=0, type=int,
31                         help='map words appearing less than threshold times to unknown')
32     parser.add_argument('--tgtdict', metavar='FP', help='reuse given target dictionary')
33     parser.add_argument('--srcdict', metavar='FP', help='reuse given source dictionary')
34     parser.add_argument('--nwordstgt', metavar='N', default=-1, type=int, help='number of target words to retain')
35     parser.add_argument('--nwordssrc', metavar='N', default=-1, type=int, help='number of source words to retain')
36     parser.add_argument('--alignfile', metavar='ALIGN', default=None, help='an alignment file (optional)')
37     parser.add_argument('--output-format', metavar='FORMAT', default='binary', choices=['binary', 'raw'],
38                         help='output format (optional)')
39 
40     args = parser.parse_args()
41     print(args)
42     os.makedirs(args.destdir, exist_ok=True)
43 
44     if args.srcdict:
45         src_dict = dictionary.Dictionary.load(args.srcdict)
46     else:
47         src_dict = Tokenizer.build_dictionary(filename='{}.{}'.format(args.trainpref, args.source_lang))
48     src_dict.save(os.path.join(args.destdir, 'dict.{}.txt'.format(args.source_lang)),
49                   threshold=args.thresholdsrc, nwords=args.nwordssrc)
50 
51     if args.tgtdict:
52         tgt_dict = dictionary.Dictionary.load(args.tgtdict)
53     else:
54         tgt_dict = Tokenizer.build_dictionary(filename='{}.{}'.format(args.trainpref, args.target_lang))
55     tgt_dict.save(os.path.join(args.destdir, 'dict.{}.txt'.format(args.target_lang)),
56                   threshold=args.thresholdtgt, nwords=args.nwordstgt)
57 
58     def make_binary_dataset(input_prefix, output_prefix, lang):
59         dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(lang)))
60         print('| [{}] Dictionary: {} types'.format(lang, len(dict) - 1))
61 
62         ds = indexed_dataset.IndexedDatasetBuilder(
63             '{}/{}.{}-{}.{}.bin'.format(args.destdir, output_prefix, args.source_lang,
64                                         args.target_lang, lang)
65         )
66 
67         def consumer(tensor):
68             ds.add_item(tensor)
69 
70         input_file = '{}.{}'.format(input_prefix, lang)
71         res = Tokenizer.binarize(input_file, dict, consumer)
72         print('| [{}] {}: {} sents, {} tokens, {:.3}% replaced by {}'.format(
73             lang, input_file, res['nseq'], res['ntok'],
74             100 * res['nunk'] / res['ntok'], dict.unk_word))
75         ds.finalize('{}/{}.{}-{}.{}.idx'.format(
76             args.destdir, output_prefix,
77             args.source_lang, args.target_lang, lang))
78 
79     def make_dataset(input_prefix, output_prefix, lang, output_format='binary'):
80         if output_format == 'binary':
81             make_binary_dataset(input_prefix, output_prefix, lang)
82         elif output_format == 'raw':
83             # Copy original text file to destination folder
84             output_text_file = os.path.join(args.destdir, f'{output_prefix}.{lang}')
85             shutil.copyfile('{}.{}'.format(input_prefix, lang), output_text_file)
86 
87     make_dataset(args.trainpref, 'train', args.source_lang, args.output_format)
88     make_dataset(args.trainpref, 'train', args.target_lang, args.output_format)
89     for k, validpref in enumerate(args.validpref.split(',')):
90         outprefix = 'valid{}'.format(k) if k > 0 else 'valid'
91         make_dataset(validpref, outprefix, args.source_lang, args.output_format)
92         make_dataset(validpref, outprefix, args.target_lang, args.output_format)
93     for k, testpref in enumerate(args.testpref.split(',')):
94         outprefix = 'test{}'.format(k) if k > 0 else 'test'
95         make_dataset(testpref, outprefix, args.source_lang, args.output_format)
96         make_dataset(testpref, outprefix, args.target_lang, args.output_format)
97     print('| Wrote preprocessed data to {}'.format(args.destdir))
98 
99     if args.alignfile:
100         src_file_name = '{}.{}'.format(args.trainpref, args.source_lang)
101         tgt_file_name = '{}.{}'.format(args.trainpref, args.target_lang)
102         src_dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(args.source_lang)))
103         tgt_dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(args.target_lang)))
104         freq_map = {}
105         with open(args.alignfile, 'r') as align_file:
106             with open(src_file_name, 'r') as src_file:
107                 with open(tgt_file_name, 'r') as tgt_file:
108                     for a, s, t in zip_longest(align_file, src_file, tgt_file):
109                         si = Tokenizer.tokenize(s, src_dict, add_if_not_exist=False)
110                         ti = Tokenizer.tokenize(t, tgt_dict, add_if_not_exist=False)
111                         ai = list(map(lambda x: tuple(x.split('-')), a.split()))
112                         for sai, tai in ai:
113                             srcidx = si[int(sai)]
114                             tgtidx = ti[int(tai)]
115                             if srcidx != src_dict.unk() and tgtidx != tgt_dict.unk():
116                                 assert srcidx != src_dict.pad()
117                                 assert srcidx != src_dict.eos()
118                                 assert tgtidx != tgt_dict.pad()
119                                 assert tgtidx != tgt_dict.eos()
120 
121                                 if srcidx not in freq_map:
122                                     freq_map[srcidx] = {}
123                                 if tgtidx not in freq_map[srcidx]:
124                                     freq_map[srcidx][tgtidx] = 1
125                                 else:
126                                     freq_map[srcidx][tgtidx] += 1
127 
128         align_dict = {}
129         for srcidx in freq_map.keys():
130             align_dict[srcidx] = max(freq_map[srcidx], key=freq_map[srcidx].get)
131 
132         with open(os.path.join(args.destdir, 'alignment.{}-{}.txt'.format(
133                 args.source_lang, args.target_lang)), 'w') as f:
134             for k, v in align_dict.items():
135                 print('{} {}'.format(src_dict[k], tgt_dict[v]), file=f)
136 
137 
138 if __name__ == '__main__':
139     main()
140
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/preprocess.py b/preprocess.py
--- a/preprocess.py
+++ b/preprocess.py
@@ -81,7 +81,7 @@
             make_binary_dataset(input_prefix, output_prefix, lang)
         elif output_format == 'raw':
             # Copy original text file to destination folder
-            output_text_file = os.path.join(args.destdir, f'{output_prefix}.{lang}')
+            output_text_file = os.path.join(args.destdir, '{}.{}'.format(output_prefix, lang))
             shutil.copyfile('{}.{}'.format(input_prefix, lang), output_text_file)
 
     make_dataset(args.trainpref, 'train', args.source_lang, args.output_format)

{"golden_diff": "diff --git a/preprocess.py b/preprocess.py\n--- a/preprocess.py\n+++ b/preprocess.py\n@@ -81,7 +81,7 @@\n             make_binary_dataset(input_prefix, output_prefix, lang)\n         elif output_format == 'raw':\n             # Copy original text file to destination folder\n-            output_text_file = os.path.join(args.destdir, f'{output_prefix}.{lang}')\n+            output_text_file = os.path.join(args.destdir, '{}.{}'.format(output_prefix, lang))\n             shutil.copyfile('{}.{}'.format(input_prefix, lang), output_text_file)\n \n     make_dataset(args.trainpref, 'train', args.source_lang, args.output_format)\n", "issue": "bad syntax in preprocess.py\nlevinth@zt-gpu-lin-1:~/fairseq-py$ python3 preprocess.py --source-lang de --target-lang en   --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test   --destdir data-bin/iwslt14.tokenized.de-en\r\n  File \"preprocess.py\", line 84\r\n    output_text_file = os.path.join(args.destdir, f'{output_prefix}.{lang}')\r\n                                                                          ^\r\nSyntaxError: invalid syntax\r\n\r\nreplacing line 84 with\r\noutput_text_file = os.path.join(args.destdir, '{output_prefix}.{lang}')\r\nappears to have fixed the issue\n", "before_files": [{"content": "#!/usr/bin/env python3\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the LICENSE file in\n# the root directory of this source tree. An additional grant of patent rights\n# can be found in the PATENTS file in the same directory.\n#\n\nimport argparse\nfrom itertools import zip_longest\nimport os\nimport shutil\n\nfrom fairseq import dictionary, indexed_dataset\nfrom fairseq.tokenizer import Tokenizer\n\n\ndef main():\n    parser = argparse.ArgumentParser(\n        description='Data pre-processing: Create dictionary and store data in binary format')\n    parser.add_argument('-s', '--source-lang', default=None, metavar='SRC', help='source language')\n    parser.add_argument('-t', '--target-lang', default=None, metavar='TARGET', help='target language')\n    parser.add_argument('--trainpref', metavar='FP', default='train', help='target language')\n    parser.add_argument('--validpref', metavar='FP', default='valid', help='comma separated, valid language prefixes')\n    parser.add_argument('--testpref', metavar='FP', default='test', help='comma separated, test language prefixes')\n    parser.add_argument('--destdir', metavar='DIR', default='data-bin', help='destination dir')\n    parser.add_argument('--thresholdtgt', metavar='N', default=0, type=int,\n                        help='map words appearing less than threshold times to unknown')\n    parser.add_argument('--thresholdsrc', metavar='N', default=0, type=int,\n                        help='map words appearing less than threshold times to unknown')\n    parser.add_argument('--tgtdict', metavar='FP', help='reuse given target dictionary')\n    parser.add_argument('--srcdict', metavar='FP', help='reuse given source dictionary')\n    parser.add_argument('--nwordstgt', metavar='N', default=-1, type=int, help='number of target words to retain')\n    parser.add_argument('--nwordssrc', metavar='N', default=-1, type=int, help='number of source words to retain')\n    parser.add_argument('--alignfile', metavar='ALIGN', default=None, help='an alignment file (optional)')\n    parser.add_argument('--output-format', metavar='FORMAT', default='binary', choices=['binary', 'raw'],\n                        help='output format (optional)')\n\n    args = parser.parse_args()\n    print(args)\n    os.makedirs(args.destdir, exist_ok=True)\n\n    if args.srcdict:\n        src_dict = dictionary.Dictionary.load(args.srcdict)\n    else:\n        src_dict = Tokenizer.build_dictionary(filename='{}.{}'.format(args.trainpref, args.source_lang))\n    src_dict.save(os.path.join(args.destdir, 'dict.{}.txt'.format(args.source_lang)),\n                  threshold=args.thresholdsrc, nwords=args.nwordssrc)\n\n    if args.tgtdict:\n        tgt_dict = dictionary.Dictionary.load(args.tgtdict)\n    else:\n        tgt_dict = Tokenizer.build_dictionary(filename='{}.{}'.format(args.trainpref, args.target_lang))\n    tgt_dict.save(os.path.join(args.destdir, 'dict.{}.txt'.format(args.target_lang)),\n                  threshold=args.thresholdtgt, nwords=args.nwordstgt)\n\n    def make_binary_dataset(input_prefix, output_prefix, lang):\n        dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(lang)))\n        print('| [{}] Dictionary: {} types'.format(lang, len(dict) - 1))\n\n        ds = indexed_dataset.IndexedDatasetBuilder(\n            '{}/{}.{}-{}.{}.bin'.format(args.destdir, output_prefix, args.source_lang,\n                                        args.target_lang, lang)\n        )\n\n        def consumer(tensor):\n            ds.add_item(tensor)\n\n        input_file = '{}.{}'.format(input_prefix, lang)\n        res = Tokenizer.binarize(input_file, dict, consumer)\n        print('| [{}] {}: {} sents, {} tokens, {:.3}% replaced by {}'.format(\n            lang, input_file, res['nseq'], res['ntok'],\n            100 * res['nunk'] / res['ntok'], dict.unk_word))\n        ds.finalize('{}/{}.{}-{}.{}.idx'.format(\n            args.destdir, output_prefix,\n            args.source_lang, args.target_lang, lang))\n\n    def make_dataset(input_prefix, output_prefix, lang, output_format='binary'):\n        if output_format == 'binary':\n            make_binary_dataset(input_prefix, output_prefix, lang)\n        elif output_format == 'raw':\n            # Copy original text file to destination folder\n            output_text_file = os.path.join(args.destdir, f'{output_prefix}.{lang}')\n            shutil.copyfile('{}.{}'.format(input_prefix, lang), output_text_file)\n\n    make_dataset(args.trainpref, 'train', args.source_lang, args.output_format)\n    make_dataset(args.trainpref, 'train', args.target_lang, args.output_format)\n    for k, validpref in enumerate(args.validpref.split(',')):\n        outprefix = 'valid{}'.format(k) if k > 0 else 'valid'\n        make_dataset(validpref, outprefix, args.source_lang, args.output_format)\n        make_dataset(validpref, outprefix, args.target_lang, args.output_format)\n    for k, testpref in enumerate(args.testpref.split(',')):\n        outprefix = 'test{}'.format(k) if k > 0 else 'test'\n        make_dataset(testpref, outprefix, args.source_lang, args.output_format)\n        make_dataset(testpref, outprefix, args.target_lang, args.output_format)\n    print('| Wrote preprocessed data to {}'.format(args.destdir))\n\n    if args.alignfile:\n        src_file_name = '{}.{}'.format(args.trainpref, args.source_lang)\n        tgt_file_name = '{}.{}'.format(args.trainpref, args.target_lang)\n        src_dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(args.source_lang)))\n        tgt_dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(args.target_lang)))\n        freq_map = {}\n        with open(args.alignfile, 'r') as align_file:\n            with open(src_file_name, 'r') as src_file:\n                with open(tgt_file_name, 'r') as tgt_file:\n                    for a, s, t in zip_longest(align_file, src_file, tgt_file):\n                        si = Tokenizer.tokenize(s, src_dict, add_if_not_exist=False)\n                        ti = Tokenizer.tokenize(t, tgt_dict, add_if_not_exist=False)\n                        ai = list(map(lambda x: tuple(x.split('-')), a.split()))\n                        for sai, tai in ai:\n                            srcidx = si[int(sai)]\n                            tgtidx = ti[int(tai)]\n                            if srcidx != src_dict.unk() and tgtidx != tgt_dict.unk():\n                                assert srcidx != src_dict.pad()\n                                assert srcidx != src_dict.eos()\n                                assert tgtidx != tgt_dict.pad()\n                                assert tgtidx != tgt_dict.eos()\n\n                                if srcidx not in freq_map:\n                                    freq_map[srcidx] = {}\n                                if tgtidx not in freq_map[srcidx]:\n                                    freq_map[srcidx][tgtidx] = 1\n                                else:\n                                    freq_map[srcidx][tgtidx] += 1\n\n        align_dict = {}\n        for srcidx in freq_map.keys():\n            align_dict[srcidx] = max(freq_map[srcidx], key=freq_map[srcidx].get)\n\n        with open(os.path.join(args.destdir, 'alignment.{}-{}.txt'.format(\n                args.source_lang, args.target_lang)), 'w') as f:\n            for k, v in align_dict.items():\n                print('{} {}'.format(src_dict[k], tgt_dict[v]), file=f)\n\n\nif __name__ == '__main__':\n    main()\n", "path": "preprocess.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the LICENSE file in\n# the root directory of this source tree. An additional grant of patent rights\n# can be found in the PATENTS file in the same directory.\n#\n\nimport argparse\nfrom itertools import zip_longest\nimport os\nimport shutil\n\nfrom fairseq import dictionary, indexed_dataset\nfrom fairseq.tokenizer import Tokenizer\n\n\ndef main():\n    parser = argparse.ArgumentParser(\n        description='Data pre-processing: Create dictionary and store data in binary format')\n    parser.add_argument('-s', '--source-lang', default=None, metavar='SRC', help='source language')\n    parser.add_argument('-t', '--target-lang', default=None, metavar='TARGET', help='target language')\n    parser.add_argument('--trainpref', metavar='FP', default='train', help='target language')\n    parser.add_argument('--validpref', metavar='FP', default='valid', help='comma separated, valid language prefixes')\n    parser.add_argument('--testpref', metavar='FP', default='test', help='comma separated, test language prefixes')\n    parser.add_argument('--destdir', metavar='DIR', default='data-bin', help='destination dir')\n    parser.add_argument('--thresholdtgt', metavar='N', default=0, type=int,\n                        help='map words appearing less than threshold times to unknown')\n    parser.add_argument('--thresholdsrc', metavar='N', default=0, type=int,\n                        help='map words appearing less than threshold times to unknown')\n    parser.add_argument('--tgtdict', metavar='FP', help='reuse given target dictionary')\n    parser.add_argument('--srcdict', metavar='FP', help='reuse given source dictionary')\n    parser.add_argument('--nwordstgt', metavar='N', default=-1, type=int, help='number of target words to retain')\n    parser.add_argument('--nwordssrc', metavar='N', default=-1, type=int, help='number of source words to retain')\n    parser.add_argument('--alignfile', metavar='ALIGN', default=None, help='an alignment file (optional)')\n    parser.add_argument('--output-format', metavar='FORMAT', default='binary', choices=['binary', 'raw'],\n                        help='output format (optional)')\n\n    args = parser.parse_args()\n    print(args)\n    os.makedirs(args.destdir, exist_ok=True)\n\n    if args.srcdict:\n        src_dict = dictionary.Dictionary.load(args.srcdict)\n    else:\n        src_dict = Tokenizer.build_dictionary(filename='{}.{}'.format(args.trainpref, args.source_lang))\n    src_dict.save(os.path.join(args.destdir, 'dict.{}.txt'.format(args.source_lang)),\n                  threshold=args.thresholdsrc, nwords=args.nwordssrc)\n\n    if args.tgtdict:\n        tgt_dict = dictionary.Dictionary.load(args.tgtdict)\n    else:\n        tgt_dict = Tokenizer.build_dictionary(filename='{}.{}'.format(args.trainpref, args.target_lang))\n    tgt_dict.save(os.path.join(args.destdir, 'dict.{}.txt'.format(args.target_lang)),\n                  threshold=args.thresholdtgt, nwords=args.nwordstgt)\n\n    def make_binary_dataset(input_prefix, output_prefix, lang):\n        dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(lang)))\n        print('| [{}] Dictionary: {} types'.format(lang, len(dict) - 1))\n\n        ds = indexed_dataset.IndexedDatasetBuilder(\n            '{}/{}.{}-{}.{}.bin'.format(args.destdir, output_prefix, args.source_lang,\n                                        args.target_lang, lang)\n        )\n\n        def consumer(tensor):\n            ds.add_item(tensor)\n\n        input_file = '{}.{}'.format(input_prefix, lang)\n        res = Tokenizer.binarize(input_file, dict, consumer)\n        print('| [{}] {}: {} sents, {} tokens, {:.3}% replaced by {}'.format(\n            lang, input_file, res['nseq'], res['ntok'],\n            100 * res['nunk'] / res['ntok'], dict.unk_word))\n        ds.finalize('{}/{}.{}-{}.{}.idx'.format(\n            args.destdir, output_prefix,\n            args.source_lang, args.target_lang, lang))\n\n    def make_dataset(input_prefix, output_prefix, lang, output_format='binary'):\n        if output_format == 'binary':\n            make_binary_dataset(input_prefix, output_prefix, lang)\n        elif output_format == 'raw':\n            # Copy original text file to destination folder\n            output_text_file = os.path.join(args.destdir, '{}.{}'.format(output_prefix, lang))\n            shutil.copyfile('{}.{}'.format(input_prefix, lang), output_text_file)\n\n    make_dataset(args.trainpref, 'train', args.source_lang, args.output_format)\n    make_dataset(args.trainpref, 'train', args.target_lang, args.output_format)\n    for k, validpref in enumerate(args.validpref.split(',')):\n        outprefix = 'valid{}'.format(k) if k > 0 else 'valid'\n        make_dataset(validpref, outprefix, args.source_lang, args.output_format)\n        make_dataset(validpref, outprefix, args.target_lang, args.output_format)\n    for k, testpref in enumerate(args.testpref.split(',')):\n        outprefix = 'test{}'.format(k) if k > 0 else 'test'\n        make_dataset(testpref, outprefix, args.source_lang, args.output_format)\n        make_dataset(testpref, outprefix, args.target_lang, args.output_format)\n    print('| Wrote preprocessed data to {}'.format(args.destdir))\n\n    if args.alignfile:\n        src_file_name = '{}.{}'.format(args.trainpref, args.source_lang)\n        tgt_file_name = '{}.{}'.format(args.trainpref, args.target_lang)\n        src_dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(args.source_lang)))\n        tgt_dict = dictionary.Dictionary.load(os.path.join(args.destdir, 'dict.{}.txt'.format(args.target_lang)))\n        freq_map = {}\n        with open(args.alignfile, 'r') as align_file:\n            with open(src_file_name, 'r') as src_file:\n                with open(tgt_file_name, 'r') as tgt_file:\n                    for a, s, t in zip_longest(align_file, src_file, tgt_file):\n                        si = Tokenizer.tokenize(s, src_dict, add_if_not_exist=False)\n                        ti = Tokenizer.tokenize(t, tgt_dict, add_if_not_exist=False)\n                        ai = list(map(lambda x: tuple(x.split('-')), a.split()))\n                        for sai, tai in ai:\n                            srcidx = si[int(sai)]\n                            tgtidx = ti[int(tai)]\n                            if srcidx != src_dict.unk() and tgtidx != tgt_dict.unk():\n                                assert srcidx != src_dict.pad()\n                                assert srcidx != src_dict.eos()\n                                assert tgtidx != tgt_dict.pad()\n                                assert tgtidx != tgt_dict.eos()\n\n                                if srcidx not in freq_map:\n                                    freq_map[srcidx] = {}\n                                if tgtidx not in freq_map[srcidx]:\n                                    freq_map[srcidx][tgtidx] = 1\n                                else:\n                                    freq_map[srcidx][tgtidx] += 1\n\n        align_dict = {}\n        for srcidx in freq_map.keys():\n            align_dict[srcidx] = max(freq_map[srcidx], key=freq_map[srcidx].get)\n\n        with open(os.path.join(args.destdir, 'alignment.{}-{}.txt'.format(\n                args.source_lang, args.target_lang)), 'w') as f:\n            for k, v in align_dict.items():\n                print('{} {}'.format(src_dict[k], tgt_dict[v]), file=f)\n\n\nif __name__ == '__main__':\n    main()\n", "path": "preprocess.py"}]}