We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
PRF evaluator: list index out of range
Hi! 
I'm getting `list index out of range` when prf evaluator is used.

**Config:**
Model: TransformerRelative
params:
  beam_width: 1

train:
  maximum_features_length: 50
  maximum_labels_length: 50
  save_summary_steps: 100
  sample_buffer_size: 1000000
  keep_checkpoint_max: 20
  save_checkpoints_steps: 5000
  max_step: 2000000

eval:
  batch_size: 32
  steps: 5000
  export_on_best: bleu
  external_evaluators: [ "bleu", "prf", "wer" ]

infer:
  batch_size: 1024

**Full stack:**
W tensorflow/core/kernels/data/generator_dataset_op.cc:103] Error occurred when finalizing GeneratorDataset iterator: Cancelled: Operation was cancelled
Traceback (most recent call last):
  File "/home/dima/anaconda3/envs/tf/bin/onmt-main", line 8, in <module>
    sys.exit(main())
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/bin/main.py", line 224, in main
    hvd=hvd)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/runner.py", line 217, in train
    moving_average_decay=train_config.get("moving_average_decay"))
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py", line 118, in __call__
    early_stop = self._evaluate(evaluator, step, moving_average=moving_average)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py", line 140, in _evaluate
    evaluator(step)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/evaluation.py", line 299, in __call__
    score = scorer(self._labels_file, output_path)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/scorers.py", line 132, in __call__
    precision_score, recall_score, fmeasure_score = fmeasure(ref_path, hyp_path)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/fmeasure.py", line 49, in fmeasure
    if tag == classref[linecpt][tagcpt]:
IndexError: list index out of range

Can I help you with  the issue? I'm not familiar with the code base, but I can try to reproduce it locally and extract the context if necessary.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `opennmt/utils/fmeasure.py`
Content:
```
1 """Hypotheses file scoring for Precision Recall and F-Measure."""
2 
3 def fmeasure(ref_path,
4              hyp_path,
5              return_precision_only=False,
6              return_recall_only=False,
7              return_fmeasure_only=False):
8   """Compute Precision Recall and F-Measure between two files"""
9   with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:
10     list_null_tags = ["X", "null", "NULL", "Null", "O"]
11     listtags = []
12     linecpt = 0
13     classref = []
14     classrandom = []
15     classhyp = []
16     nbrtagref = {}
17     nbrtaghyp = {}
18     nbrtagok = {}
19     for tag in listtags:
20       nbrtagref[tag] = 0
21       nbrtaghyp[tag] = 0
22       nbrtagok[tag] = 0
23     for line in ref_fp:
24       line = line.strip()
25       tabline = line.split(' ')
26       tagcpt = 0
27       lineref = []
28       for tag in tabline:
29         lineref.append(tag)
30         if tag in nbrtagref.keys() and tag not in list_null_tags:
31           nbrtagref[tag] = nbrtagref[tag]+1
32         else:
33           nbrtagref[tag] = 1
34         tagcpt = tagcpt+1
35       classref.append(lineref)
36       linecpt = linecpt+1
37     linecpt = 0
38     for line in hyp_fp:
39       line = line.strip()
40       tabline = line.split(' ')
41       tagcpt = 0
42       linehyp = []
43       linerandom = []
44       for tag in tabline:
45         linehyp.append(tag)
46         if tag not in listtags:
47           listtags.append(tag)
48         linerandom.append(tag)
49         if tag == classref[linecpt][tagcpt]:
50           if tag in nbrtagok.keys():
51             nbrtagok[tag] = nbrtagok[tag]+1
52           else:
53             nbrtagok[tag] = 1
54         tagcpt = tagcpt+1
55         if tag in nbrtaghyp.keys():
56           nbrtaghyp[tag] = nbrtaghyp[tag]+1
57         else:
58           nbrtaghyp[tag] = 1
59       classhyp.append(linehyp)
60       classrandom.append(linerandom)
61       linecpt = linecpt+1
62 
63   tagcpt = 0
64   fullprecision = 0
65   fullrecall = 0
66   precision = {}
67   recall = {}
68   fulltagok = 0.00
69   fulltaghyp = 0.00
70   fulltagref = 0.00
71   for tag in listtags:
72     if tag not in nbrtagok:
73       nbrtagok[tag] = 0
74     if tag not in nbrtaghyp:
75       nbrtaghyp[tag] = 0
76     if tag not in nbrtagref:
77       nbrtagref[tag] = 0
78     if nbrtaghyp[tag] != 0:
79       precision[tag] = nbrtagok[tag]/nbrtaghyp[tag]
80     else:
81       precision[tag] = 0
82     if nbrtagref[tag] != 0:
83       recall[tag] = nbrtagok[tag]/nbrtagref[tag]
84     else:
85       recall[tag] = 0
86     if tag not in list_null_tags:
87       fulltagok = fulltagok+nbrtagok[tag]
88       fulltaghyp = fulltaghyp+nbrtaghyp[tag]
89       fulltagref = fulltagref+nbrtagref[tag]
90 #      fullprecision = fullprecision+precision[tag]
91 #      fullrecall = fullrecall+recall[tag]
92     tagcpt = tagcpt+1
93   fullprecision = round(100*fulltagok/fulltaghyp, 2)/100
94   fullrecall = round(100*fulltagok/fulltagref, 2)/100
95   fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100
96   if return_precision_only:
97     return fullprecision
98   if return_recall_only:
99     return fullrecall
100   if return_fmeasure_only:
101     return fullfmeasure
102   return fullprecision, fullrecall, fullfmeasure
103
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/opennmt/utils/fmeasure.py b/opennmt/utils/fmeasure.py
--- a/opennmt/utils/fmeasure.py
+++ b/opennmt/utils/fmeasure.py
@@ -9,21 +9,15 @@
   with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:
     list_null_tags = ["X", "null", "NULL", "Null", "O"]
     listtags = []
-    linecpt = 0
     classref = []
     classrandom = []
     classhyp = []
     nbrtagref = {}
     nbrtaghyp = {}
     nbrtagok = {}
-    for tag in listtags:
-      nbrtagref[tag] = 0
-      nbrtaghyp[tag] = 0
-      nbrtagok[tag] = 0
     for line in ref_fp:
       line = line.strip()
       tabline = line.split(' ')
-      tagcpt = 0
       lineref = []
       for tag in tabline:
         lineref.append(tag)
@@ -31,36 +25,29 @@
           nbrtagref[tag] = nbrtagref[tag]+1
         else:
           nbrtagref[tag] = 1
-        tagcpt = tagcpt+1
       classref.append(lineref)
-      linecpt = linecpt+1
-    linecpt = 0
-    for line in hyp_fp:
+    for line, lineref in zip(hyp_fp, classref):
       line = line.strip()
       tabline = line.split(' ')
-      tagcpt = 0
       linehyp = []
       linerandom = []
-      for tag in tabline:
+      for tagcpt, tag in enumerate(tabline):
         linehyp.append(tag)
         if tag not in listtags:
           listtags.append(tag)
         linerandom.append(tag)
-        if tag == classref[linecpt][tagcpt]:
+        if tagcpt < len(lineref) and tag == lineref[tagcpt]:
           if tag in nbrtagok.keys():
             nbrtagok[tag] = nbrtagok[tag]+1
           else:
             nbrtagok[tag] = 1
-        tagcpt = tagcpt+1
         if tag in nbrtaghyp.keys():
           nbrtaghyp[tag] = nbrtaghyp[tag]+1
         else:
           nbrtaghyp[tag] = 1
       classhyp.append(linehyp)
       classrandom.append(linerandom)
-      linecpt = linecpt+1
 
-  tagcpt = 0
   fullprecision = 0
   fullrecall = 0
   precision = {}
@@ -87,12 +74,11 @@
       fulltagok = fulltagok+nbrtagok[tag]
       fulltaghyp = fulltaghyp+nbrtaghyp[tag]
       fulltagref = fulltagref+nbrtagref[tag]
-#      fullprecision = fullprecision+precision[tag]
-#      fullrecall = fullrecall+recall[tag]
-    tagcpt = tagcpt+1
-  fullprecision = round(100*fulltagok/fulltaghyp, 2)/100
-  fullrecall = round(100*fulltagok/fulltagref, 2)/100
-  fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100
+  fullprecision = fulltagok / fulltaghyp if fulltaghyp != 0 else 0
+  fullrecall = fulltagok / fulltagref if fulltagref != 0 else 0
+  fullfmeasure = (
+      (2 * fullprecision * fullrecall) / (fullprecision + fullrecall)
+      if (fullprecision + fullrecall) != 0 else 0)
   if return_precision_only:
     return fullprecision
   if return_recall_only:

{"golden_diff": "diff --git a/opennmt/utils/fmeasure.py b/opennmt/utils/fmeasure.py\n--- a/opennmt/utils/fmeasure.py\n+++ b/opennmt/utils/fmeasure.py\n@@ -9,21 +9,15 @@\n   with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:\n     list_null_tags = [\"X\", \"null\", \"NULL\", \"Null\", \"O\"]\n     listtags = []\n-    linecpt = 0\n     classref = []\n     classrandom = []\n     classhyp = []\n     nbrtagref = {}\n     nbrtaghyp = {}\n     nbrtagok = {}\n-    for tag in listtags:\n-      nbrtagref[tag] = 0\n-      nbrtaghyp[tag] = 0\n-      nbrtagok[tag] = 0\n     for line in ref_fp:\n       line = line.strip()\n       tabline = line.split(' ')\n-      tagcpt = 0\n       lineref = []\n       for tag in tabline:\n         lineref.append(tag)\n@@ -31,36 +25,29 @@\n           nbrtagref[tag] = nbrtagref[tag]+1\n         else:\n           nbrtagref[tag] = 1\n-        tagcpt = tagcpt+1\n       classref.append(lineref)\n-      linecpt = linecpt+1\n-    linecpt = 0\n-    for line in hyp_fp:\n+    for line, lineref in zip(hyp_fp, classref):\n       line = line.strip()\n       tabline = line.split(' ')\n-      tagcpt = 0\n       linehyp = []\n       linerandom = []\n-      for tag in tabline:\n+      for tagcpt, tag in enumerate(tabline):\n         linehyp.append(tag)\n         if tag not in listtags:\n           listtags.append(tag)\n         linerandom.append(tag)\n-        if tag == classref[linecpt][tagcpt]:\n+        if tagcpt < len(lineref) and tag == lineref[tagcpt]:\n           if tag in nbrtagok.keys():\n             nbrtagok[tag] = nbrtagok[tag]+1\n           else:\n             nbrtagok[tag] = 1\n-        tagcpt = tagcpt+1\n         if tag in nbrtaghyp.keys():\n           nbrtaghyp[tag] = nbrtaghyp[tag]+1\n         else:\n           nbrtaghyp[tag] = 1\n       classhyp.append(linehyp)\n       classrandom.append(linerandom)\n-      linecpt = linecpt+1\n \n-  tagcpt = 0\n   fullprecision = 0\n   fullrecall = 0\n   precision = {}\n@@ -87,12 +74,11 @@\n       fulltagok = fulltagok+nbrtagok[tag]\n       fulltaghyp = fulltaghyp+nbrtaghyp[tag]\n       fulltagref = fulltagref+nbrtagref[tag]\n-#      fullprecision = fullprecision+precision[tag]\n-#      fullrecall = fullrecall+recall[tag]\n-    tagcpt = tagcpt+1\n-  fullprecision = round(100*fulltagok/fulltaghyp, 2)/100\n-  fullrecall = round(100*fulltagok/fulltagref, 2)/100\n-  fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100\n+  fullprecision = fulltagok / fulltaghyp if fulltaghyp != 0 else 0\n+  fullrecall = fulltagok / fulltagref if fulltagref != 0 else 0\n+  fullfmeasure = (\n+      (2 * fullprecision * fullrecall) / (fullprecision + fullrecall)\n+      if (fullprecision + fullrecall) != 0 else 0)\n   if return_precision_only:\n     return fullprecision\n   if return_recall_only:\n", "issue": "PRF evaluator: list index out of range\nHi! \r\nI'm getting `list index out of range` when prf evaluator is used.\r\n\r\n**Config:**\r\nModel: TransformerRelative\r\nparams:\r\n  beam_width: 1\r\n\r\ntrain:\r\n  maximum_features_length: 50\r\n  maximum_labels_length: 50\r\n  save_summary_steps: 100\r\n  sample_buffer_size: 1000000\r\n  keep_checkpoint_max: 20\r\n  save_checkpoints_steps: 5000\r\n  max_step: 2000000\r\n\r\neval:\r\n  batch_size: 32\r\n  steps: 5000\r\n  export_on_best: bleu\r\n  external_evaluators: [ \"bleu\", \"prf\", \"wer\" ]\r\n\r\ninfer:\r\n  batch_size: 1024\r\n\r\n**Full stack:**\r\nW tensorflow/core/kernels/data/generator_dataset_op.cc:103] Error occurred when finalizing GeneratorDataset iterator: Cancelled: Operation was cancelled\r\nTraceback (most recent call last):\r\n  File \"/home/dima/anaconda3/envs/tf/bin/onmt-main\", line 8, in <module>\r\n    sys.exit(main())\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/bin/main.py\", line 224, in main\r\n    hvd=hvd)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/runner.py\", line 217, in train\r\n    moving_average_decay=train_config.get(\"moving_average_decay\"))\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py\", line 118, in __call__\r\n    early_stop = self._evaluate(evaluator, step, moving_average=moving_average)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py\", line 140, in _evaluate\r\n    evaluator(step)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/evaluation.py\", line 299, in __call__\r\n    score = scorer(self._labels_file, output_path)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/scorers.py\", line 132, in __call__\r\n    precision_score, recall_score, fmeasure_score = fmeasure(ref_path, hyp_path)\r\n  File \"/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/fmeasure.py\", line 49, in fmeasure\r\n    if tag == classref[linecpt][tagcpt]:\r\nIndexError: list index out of range\r\n\r\nCan I help you with  the issue? I'm not familiar with the code base, but I can try to reproduce it locally and extract the context if necessary.\r\n\n", "before_files": [{"content": "\"\"\"Hypotheses file scoring for Precision Recall and F-Measure.\"\"\"\n\ndef fmeasure(ref_path,\n             hyp_path,\n             return_precision_only=False,\n             return_recall_only=False,\n             return_fmeasure_only=False):\n  \"\"\"Compute Precision Recall and F-Measure between two files\"\"\"\n  with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:\n    list_null_tags = [\"X\", \"null\", \"NULL\", \"Null\", \"O\"]\n    listtags = []\n    linecpt = 0\n    classref = []\n    classrandom = []\n    classhyp = []\n    nbrtagref = {}\n    nbrtaghyp = {}\n    nbrtagok = {}\n    for tag in listtags:\n      nbrtagref[tag] = 0\n      nbrtaghyp[tag] = 0\n      nbrtagok[tag] = 0\n    for line in ref_fp:\n      line = line.strip()\n      tabline = line.split(' ')\n      tagcpt = 0\n      lineref = []\n      for tag in tabline:\n        lineref.append(tag)\n        if tag in nbrtagref.keys() and tag not in list_null_tags:\n          nbrtagref[tag] = nbrtagref[tag]+1\n        else:\n          nbrtagref[tag] = 1\n        tagcpt = tagcpt+1\n      classref.append(lineref)\n      linecpt = linecpt+1\n    linecpt = 0\n    for line in hyp_fp:\n      line = line.strip()\n      tabline = line.split(' ')\n      tagcpt = 0\n      linehyp = []\n      linerandom = []\n      for tag in tabline:\n        linehyp.append(tag)\n        if tag not in listtags:\n          listtags.append(tag)\n        linerandom.append(tag)\n        if tag == classref[linecpt][tagcpt]:\n          if tag in nbrtagok.keys():\n            nbrtagok[tag] = nbrtagok[tag]+1\n          else:\n            nbrtagok[tag] = 1\n        tagcpt = tagcpt+1\n        if tag in nbrtaghyp.keys():\n          nbrtaghyp[tag] = nbrtaghyp[tag]+1\n        else:\n          nbrtaghyp[tag] = 1\n      classhyp.append(linehyp)\n      classrandom.append(linerandom)\n      linecpt = linecpt+1\n\n  tagcpt = 0\n  fullprecision = 0\n  fullrecall = 0\n  precision = {}\n  recall = {}\n  fulltagok = 0.00\n  fulltaghyp = 0.00\n  fulltagref = 0.00\n  for tag in listtags:\n    if tag not in nbrtagok:\n      nbrtagok[tag] = 0\n    if tag not in nbrtaghyp:\n      nbrtaghyp[tag] = 0\n    if tag not in nbrtagref:\n      nbrtagref[tag] = 0\n    if nbrtaghyp[tag] != 0:\n      precision[tag] = nbrtagok[tag]/nbrtaghyp[tag]\n    else:\n      precision[tag] = 0\n    if nbrtagref[tag] != 0:\n      recall[tag] = nbrtagok[tag]/nbrtagref[tag]\n    else:\n      recall[tag] = 0\n    if tag not in list_null_tags:\n      fulltagok = fulltagok+nbrtagok[tag]\n      fulltaghyp = fulltaghyp+nbrtaghyp[tag]\n      fulltagref = fulltagref+nbrtagref[tag]\n#      fullprecision = fullprecision+precision[tag]\n#      fullrecall = fullrecall+recall[tag]\n    tagcpt = tagcpt+1\n  fullprecision = round(100*fulltagok/fulltaghyp, 2)/100\n  fullrecall = round(100*fulltagok/fulltagref, 2)/100\n  fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100\n  if return_precision_only:\n    return fullprecision\n  if return_recall_only:\n    return fullrecall\n  if return_fmeasure_only:\n    return fullfmeasure\n  return fullprecision, fullrecall, fullfmeasure\n", "path": "opennmt/utils/fmeasure.py"}], "after_files": [{"content": "\"\"\"Hypotheses file scoring for Precision Recall and F-Measure.\"\"\"\n\ndef fmeasure(ref_path,\n             hyp_path,\n             return_precision_only=False,\n             return_recall_only=False,\n             return_fmeasure_only=False):\n  \"\"\"Compute Precision Recall and F-Measure between two files\"\"\"\n  with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:\n    list_null_tags = [\"X\", \"null\", \"NULL\", \"Null\", \"O\"]\n    listtags = []\n    classref = []\n    classrandom = []\n    classhyp = []\n    nbrtagref = {}\n    nbrtaghyp = {}\n    nbrtagok = {}\n    for line in ref_fp:\n      line = line.strip()\n      tabline = line.split(' ')\n      lineref = []\n      for tag in tabline:\n        lineref.append(tag)\n        if tag in nbrtagref.keys() and tag not in list_null_tags:\n          nbrtagref[tag] = nbrtagref[tag]+1\n        else:\n          nbrtagref[tag] = 1\n      classref.append(lineref)\n    for line, lineref in zip(hyp_fp, classref):\n      line = line.strip()\n      tabline = line.split(' ')\n      linehyp = []\n      linerandom = []\n      for tagcpt, tag in enumerate(tabline):\n        linehyp.append(tag)\n        if tag not in listtags:\n          listtags.append(tag)\n        linerandom.append(tag)\n        if tagcpt < len(lineref) and tag == lineref[tagcpt]:\n          if tag in nbrtagok.keys():\n            nbrtagok[tag] = nbrtagok[tag]+1\n          else:\n            nbrtagok[tag] = 1\n        if tag in nbrtaghyp.keys():\n          nbrtaghyp[tag] = nbrtaghyp[tag]+1\n        else:\n          nbrtaghyp[tag] = 1\n      classhyp.append(linehyp)\n      classrandom.append(linerandom)\n\n  fullprecision = 0\n  fullrecall = 0\n  precision = {}\n  recall = {}\n  fulltagok = 0.00\n  fulltaghyp = 0.00\n  fulltagref = 0.00\n  for tag in listtags:\n    if tag not in nbrtagok:\n      nbrtagok[tag] = 0\n    if tag not in nbrtaghyp:\n      nbrtaghyp[tag] = 0\n    if tag not in nbrtagref:\n      nbrtagref[tag] = 0\n    if nbrtaghyp[tag] != 0:\n      precision[tag] = nbrtagok[tag]/nbrtaghyp[tag]\n    else:\n      precision[tag] = 0\n    if nbrtagref[tag] != 0:\n      recall[tag] = nbrtagok[tag]/nbrtagref[tag]\n    else:\n      recall[tag] = 0\n    if tag not in list_null_tags:\n      fulltagok = fulltagok+nbrtagok[tag]\n      fulltaghyp = fulltaghyp+nbrtaghyp[tag]\n      fulltagref = fulltagref+nbrtagref[tag]\n  fullprecision = fulltagok / fulltaghyp if fulltaghyp != 0 else 0\n  fullrecall = fulltagok / fulltagref if fulltagref != 0 else 0\n  fullfmeasure = (\n      (2 * fullprecision * fullrecall) / (fullprecision + fullrecall)\n      if (fullprecision + fullrecall) != 0 else 0)\n  if return_precision_only:\n    return fullprecision\n  if return_recall_only:\n    return fullrecall\n  if return_fmeasure_only:\n    return fullfmeasure\n  return fullprecision, fullrecall, fullfmeasure\n", "path": "opennmt/utils/fmeasure.py"}]}