We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
How to support PPL
Can you add the PPL measurement?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `opennmt/evaluation.py`
Content:
```
1 """Evaluation related classes and functions."""
2 
3 import collections
4 import os
5 import six
6 
7 import tensorflow as tf
8 
9 from opennmt.data import dataset as dataset_lib
10 from opennmt.utils import misc
11 from opennmt.utils import scorers as scorers_lib
12 
13 
14 _SUMMARIES_SCOPE = "metrics"
15 
16 
17 class EarlyStopping(
18     collections.namedtuple("EarlyStopping",
19                            ("metric", "min_improvement", "steps"))):
20   """Conditions for early stopping."""
21 
22 
23 class Evaluator(object):
24   """Model evaluator."""
25 
26   def __init__(self,
27                model,
28                features_file,
29                labels_file,
30                batch_size,
31                scorers=None,
32                save_predictions=False,
33                early_stopping=None,
34                eval_dir=None):
35     """Initializes the evaluator.
36 
37     Args:
38       model: A :class:`opennmt.models.model.Model` to evaluate.
39       features_file: Path to the evaluation features.
40       labels_file: Path to the evaluation labels.
41       batch_size: The evaluation batch size.
42       scorers: A list of scorers, callables taking the path to the reference and
43         the hypothesis and return one or more scores.
44       save_predictions: Save evaluation predictions to a file. This is ``True``
45         when :obj:`external_evaluator` is set.
46       early_stopping: An ``EarlyStopping`` instance.
47       eval_dir: Directory where predictions can be saved.
48 
49     Raises:
50       ValueError: If predictions should be saved but the model is not compatible.
51       ValueError: If predictions should be saved but :obj:`eval_dir` is ``None``.
52       ValueError: If the :obj:`early_stopping` configuration is invalid.
53     """
54     if scorers is None:
55       scorers = []
56     if scorers:
57       save_predictions = True
58     if save_predictions:
59       if model.unsupervised:
60         raise ValueError("This model does not support saving evaluation predictions")
61       if eval_dir is None:
62         raise ValueError("Saving evaluation predictions requires eval_dir to be set")
63       if not tf.io.gfile.exists(eval_dir):
64         tf.io.gfile.makedirs(eval_dir)
65     self._model = model
66     self._labels_file = labels_file
67     self._save_predictions = save_predictions
68     self._scorers = scorers
69     self._eval_dir = eval_dir
70     self._metrics_history = []
71     if eval_dir is not None:
72       self._summary_writer = tf.summary.create_file_writer(eval_dir)
73       summaries = misc.read_summaries(eval_dir)
74       for step, values in summaries:
75         metrics = misc.extract_prefixed_keys(values, _SUMMARIES_SCOPE + "/")
76         self._metrics_history.append((step, metrics))
77     else:
78       self._summary_writer = tf.summary.create_noop_writer()
79     dataset = model.examples_inputter.make_evaluation_dataset(
80         features_file,
81         labels_file,
82         batch_size,
83         num_threads=1,
84         prefetch_buffer_size=1)
85 
86     @dataset_lib.function_on_next(dataset)
87     def _eval(next_fn):
88       source, target = next_fn()
89       outputs, predictions = model(source, labels=target)
90       loss = model.compute_loss(outputs, target, training=False)
91       return loss, predictions, target
92 
93     self._eval = _eval
94 
95     self._metrics_name = {"loss"}
96     for scorer in self._scorers:
97       self._metrics_name.update(scorer.scores_name)
98     model_metrics = self._model.get_metrics()
99     if model_metrics:
100       self._metrics_name.update(set(six.iterkeys(model_metrics)))
101 
102     if early_stopping is not None:
103       if early_stopping.metric not in self._metrics_name:
104         raise ValueError("Invalid early stopping metric '%s', expected one in %s" % (
105             early_stopping.metric, str(self._metrics_name)))
106       if early_stopping.steps <= 0:
107         raise ValueError("Early stopping steps should greater than 0")
108     self._early_stopping = early_stopping
109 
110   @classmethod
111   def from_config(cls, model, config, features_file=None, labels_file=None):
112     """Creates an evaluator from the configuration.
113 
114     Args:
115       model: A :class:`opennmt.models.model.Model` to evaluate.
116       config: The global user configuration.
117       features_file: Optional input features file to evaluate. If not set, will
118         load ``eval_features_file`` from the data configuration.
119       labels_file: Optional output labels file to evaluate. If not set, will load
120         ``eval_labels_file`` from the data configuration.
121 
122     Returns:
123       A :class:`opennmt.evaluation.Evaluator` instance.
124 
125     Raises:
126       ValueError: if one of :obj:`features_file` and :obj:`labels_file` is set
127         but not the other.
128     """
129     if (features_file is None) != (labels_file is None):
130       raise ValueError("features_file and labels_file should be both set for evaluation")
131     scorers = config["eval"].get("external_evaluators")
132     if scorers is not None:
133       scorers = scorers_lib.make_scorers(scorers)
134     early_stopping_config = config["eval"].get("early_stopping")
135     if early_stopping_config is not None:
136       early_stopping = EarlyStopping(
137           metric=early_stopping_config.get("metric", "loss"),
138           min_improvement=early_stopping_config.get("min_improvement", 0),
139           steps=early_stopping_config["steps"])
140     else:
141       early_stopping = None
142     return cls(
143         model,
144         features_file or config["data"]["eval_features_file"],
145         labels_file or config["data"].get("eval_labels_file"),
146         config["eval"]["batch_size"],
147         scorers=scorers,
148         save_predictions=config["eval"].get("save_eval_predictions", False),
149         early_stopping=early_stopping,
150         eval_dir=os.path.join(config["model_dir"], "eval"))
151 
152   @property
153   def metrics_name(self):
154     """The name of the metrics returned by this evaluator."""
155     return self._metrics_name
156 
157   @property
158   def metrics_history(self):
159     """The history of metrics result per evaluation step."""
160     return self._metrics_history
161 
162   def should_stop(self):
163     """Returns ``True`` if early stopping conditions are met."""
164     if self._early_stopping is None:
165       return False
166     target_metric = self._early_stopping.metric
167     higher_is_better = None
168     # Look if target_metric is produced by a scorer as they define the scores order.
169     for scorer in self._scorers:
170       if target_metric in scorer.scores_name:
171         higher_is_better = scorer.higher_is_better()
172         break
173     if higher_is_better is None:
174       # TODO: the condition below is not always true, find a way to set it
175       # correctly for Keras metrics.
176       higher_is_better = target_metric != "loss"
177     metrics = [values[target_metric] for _, values in self._metrics_history]
178     should_stop = early_stop(
179         metrics,
180         self._early_stopping.steps,
181         min_improvement=self._early_stopping.min_improvement,
182         higher_is_better=higher_is_better)
183     if should_stop:
184       tf.get_logger().warning(
185           "Evaluation metric '%s' did not improve more than %f in the last %d evaluations",
186           target_metric,
187           self._early_stopping.min_improvement,
188           self._early_stopping.steps)
189     return should_stop
190 
191   def __call__(self, step):
192     """Runs the evaluator.
193 
194     Args:
195       step: The current training step.
196 
197     Returns:
198       A dictionary of evaluation metrics.
199     """
200     tf.get_logger().info("Running evaluation for step %d", step)
201     output_file = None
202     output_path = None
203     if self._save_predictions:
204       output_path = os.path.join(self._eval_dir, "predictions.txt.%d" % step)
205       output_file = tf.io.gfile.GFile(output_path, "w")
206 
207     loss_num = 0
208     loss_den = 0
209     metrics = self._model.get_metrics()
210     for loss, predictions, target in self._eval():  # pylint: disable=no-value-for-parameter
211       if isinstance(loss, tuple):
212         loss_num += loss[0]
213         loss_den += loss[1]
214       else:
215         loss_num += loss
216         loss_den += 1
217       if metrics:
218         self._model.update_metrics(metrics, predictions, target)
219       if output_file is not None:
220         predictions = {k:v.numpy() for k, v in six.iteritems(predictions)}
221         for prediction in misc.extract_batches(predictions):
222           self._model.print_prediction(prediction, stream=output_file)
223     if loss_den == 0:
224       raise RuntimeError("No examples were evaluated")
225     loss = loss_num / loss_den
226 
227     results = dict(loss=loss)
228     if metrics:
229       for name, metric in six.iteritems(metrics):
230         results[name] = metric.result()
231     if self._save_predictions:
232       tf.get_logger().info("Evaluation predictions saved to %s", output_path)
233       output_file.close()
234       for scorer in self._scorers:
235         score = scorer(self._labels_file, output_path)
236         if isinstance(score, dict):
237           results.update(score)
238         else:
239           results[scorer.name] = score
240 
241     return self._record_results(step, results)
242 
243   def _record_results(self, step, results):
244     for name, value in six.iteritems(results):
245       if isinstance(value, tf.Tensor):
246         results[name] = value.numpy()
247     # Clear history for steps that are greater than step.
248     while self._metrics_history and self._metrics_history[-1][0] > step:
249       self._metrics_history.pop()
250     self._metrics_history.append((step, dict(results)))
251     tf.get_logger().info(
252         "Evaluation result for step %d: %s",
253         step,
254         " ; ".join("%s = %f" % (k, v) for k, v in six.iteritems(results)))
255     with self._summary_writer.as_default():
256       for key, value in six.iteritems(results):
257         tf.summary.scalar("%s/%s" % (_SUMMARIES_SCOPE, key), value, step=step)
258       self._summary_writer.flush()
259     return results
260 
261 
262 def early_stop(metrics, steps, min_improvement=0, higher_is_better=False):
263   """Early stopping condition.
264 
265   Args:
266     metrics: A list of metric values.
267     steps: Consider the improvement over this many steps.
268     min_improvement: Continue if the metric improved less than this value:
269     higher_is_better: Whether a higher value is better for this metric.
270 
271   Returns:
272     A boolean.
273   """
274   if len(metrics) < steps + 1:
275     return False
276 
277   def _did_improve(ref, new):
278     # Returns True if new is improving on ref.
279     if higher_is_better:
280       return new > ref + min_improvement
281     else:
282       return new < ref - min_improvement
283 
284   samples = metrics[-steps - 1:]
285 
286   # First check if the boundaries show an improvement.
287   if _did_improve(samples[0], samples[-1]):
288     return False
289 
290   # If not, only early stop if each successive evaluation did not improve.
291   for metric, next_metric in zip(samples[:-1], samples[1:]):
292     if _did_improve(metric, next_metric):
293       return False
294   return True
295
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/opennmt/evaluation.py b/opennmt/evaluation.py
--- a/opennmt/evaluation.py
+++ b/opennmt/evaluation.py
@@ -92,7 +92,7 @@
 
     self._eval = _eval
 
-    self._metrics_name = {"loss"}
+    self._metrics_name = {"loss", "perplexity"}
     for scorer in self._scorers:
       self._metrics_name.update(scorer.scores_name)
     model_metrics = self._model.get_metrics()
@@ -173,7 +173,7 @@
     if higher_is_better is None:
       # TODO: the condition below is not always true, find a way to set it
       # correctly for Keras metrics.
-      higher_is_better = target_metric != "loss"
+      higher_is_better = target_metric not in ("loss", "perplexity")
     metrics = [values[target_metric] for _, values in self._metrics_history]
     should_stop = early_stop(
         metrics,
@@ -224,7 +224,7 @@
       raise RuntimeError("No examples were evaluated")
     loss = loss_num / loss_den
 
-    results = dict(loss=loss)
+    results = dict(loss=loss, perplexity=tf.math.exp(loss))
     if metrics:
       for name, metric in six.iteritems(metrics):
         results[name] = metric.result()

{"golden_diff": "diff --git a/opennmt/evaluation.py b/opennmt/evaluation.py\n--- a/opennmt/evaluation.py\n+++ b/opennmt/evaluation.py\n@@ -92,7 +92,7 @@\n \n     self._eval = _eval\n \n-    self._metrics_name = {\"loss\"}\n+    self._metrics_name = {\"loss\", \"perplexity\"}\n     for scorer in self._scorers:\n       self._metrics_name.update(scorer.scores_name)\n     model_metrics = self._model.get_metrics()\n@@ -173,7 +173,7 @@\n     if higher_is_better is None:\n       # TODO: the condition below is not always true, find a way to set it\n       # correctly for Keras metrics.\n-      higher_is_better = target_metric != \"loss\"\n+      higher_is_better = target_metric not in (\"loss\", \"perplexity\")\n     metrics = [values[target_metric] for _, values in self._metrics_history]\n     should_stop = early_stop(\n         metrics,\n@@ -224,7 +224,7 @@\n       raise RuntimeError(\"No examples were evaluated\")\n     loss = loss_num / loss_den\n \n-    results = dict(loss=loss)\n+    results = dict(loss=loss, perplexity=tf.math.exp(loss))\n     if metrics:\n       for name, metric in six.iteritems(metrics):\n         results[name] = metric.result()\n", "issue": "How to support PPL\nCan you add the PPL measurement?\n", "before_files": [{"content": "\"\"\"Evaluation related classes and functions.\"\"\"\n\nimport collections\nimport os\nimport six\n\nimport tensorflow as tf\n\nfrom opennmt.data import dataset as dataset_lib\nfrom opennmt.utils import misc\nfrom opennmt.utils import scorers as scorers_lib\n\n\n_SUMMARIES_SCOPE = \"metrics\"\n\n\nclass EarlyStopping(\n    collections.namedtuple(\"EarlyStopping\",\n                           (\"metric\", \"min_improvement\", \"steps\"))):\n  \"\"\"Conditions for early stopping.\"\"\"\n\n\nclass Evaluator(object):\n  \"\"\"Model evaluator.\"\"\"\n\n  def __init__(self,\n               model,\n               features_file,\n               labels_file,\n               batch_size,\n               scorers=None,\n               save_predictions=False,\n               early_stopping=None,\n               eval_dir=None):\n    \"\"\"Initializes the evaluator.\n\n    Args:\n      model: A :class:`opennmt.models.model.Model` to evaluate.\n      features_file: Path to the evaluation features.\n      labels_file: Path to the evaluation labels.\n      batch_size: The evaluation batch size.\n      scorers: A list of scorers, callables taking the path to the reference and\n        the hypothesis and return one or more scores.\n      save_predictions: Save evaluation predictions to a file. This is ``True``\n        when :obj:`external_evaluator` is set.\n      early_stopping: An ``EarlyStopping`` instance.\n      eval_dir: Directory where predictions can be saved.\n\n    Raises:\n      ValueError: If predictions should be saved but the model is not compatible.\n      ValueError: If predictions should be saved but :obj:`eval_dir` is ``None``.\n      ValueError: If the :obj:`early_stopping` configuration is invalid.\n    \"\"\"\n    if scorers is None:\n      scorers = []\n    if scorers:\n      save_predictions = True\n    if save_predictions:\n      if model.unsupervised:\n        raise ValueError(\"This model does not support saving evaluation predictions\")\n      if eval_dir is None:\n        raise ValueError(\"Saving evaluation predictions requires eval_dir to be set\")\n      if not tf.io.gfile.exists(eval_dir):\n        tf.io.gfile.makedirs(eval_dir)\n    self._model = model\n    self._labels_file = labels_file\n    self._save_predictions = save_predictions\n    self._scorers = scorers\n    self._eval_dir = eval_dir\n    self._metrics_history = []\n    if eval_dir is not None:\n      self._summary_writer = tf.summary.create_file_writer(eval_dir)\n      summaries = misc.read_summaries(eval_dir)\n      for step, values in summaries:\n        metrics = misc.extract_prefixed_keys(values, _SUMMARIES_SCOPE + \"/\")\n        self._metrics_history.append((step, metrics))\n    else:\n      self._summary_writer = tf.summary.create_noop_writer()\n    dataset = model.examples_inputter.make_evaluation_dataset(\n        features_file,\n        labels_file,\n        batch_size,\n        num_threads=1,\n        prefetch_buffer_size=1)\n\n    @dataset_lib.function_on_next(dataset)\n    def _eval(next_fn):\n      source, target = next_fn()\n      outputs, predictions = model(source, labels=target)\n      loss = model.compute_loss(outputs, target, training=False)\n      return loss, predictions, target\n\n    self._eval = _eval\n\n    self._metrics_name = {\"loss\"}\n    for scorer in self._scorers:\n      self._metrics_name.update(scorer.scores_name)\n    model_metrics = self._model.get_metrics()\n    if model_metrics:\n      self._metrics_name.update(set(six.iterkeys(model_metrics)))\n\n    if early_stopping is not None:\n      if early_stopping.metric not in self._metrics_name:\n        raise ValueError(\"Invalid early stopping metric '%s', expected one in %s\" % (\n            early_stopping.metric, str(self._metrics_name)))\n      if early_stopping.steps <= 0:\n        raise ValueError(\"Early stopping steps should greater than 0\")\n    self._early_stopping = early_stopping\n\n  @classmethod\n  def from_config(cls, model, config, features_file=None, labels_file=None):\n    \"\"\"Creates an evaluator from the configuration.\n\n    Args:\n      model: A :class:`opennmt.models.model.Model` to evaluate.\n      config: The global user configuration.\n      features_file: Optional input features file to evaluate. If not set, will\n        load ``eval_features_file`` from the data configuration.\n      labels_file: Optional output labels file to evaluate. If not set, will load\n        ``eval_labels_file`` from the data configuration.\n\n    Returns:\n      A :class:`opennmt.evaluation.Evaluator` instance.\n\n    Raises:\n      ValueError: if one of :obj:`features_file` and :obj:`labels_file` is set\n        but not the other.\n    \"\"\"\n    if (features_file is None) != (labels_file is None):\n      raise ValueError(\"features_file and labels_file should be both set for evaluation\")\n    scorers = config[\"eval\"].get(\"external_evaluators\")\n    if scorers is not None:\n      scorers = scorers_lib.make_scorers(scorers)\n    early_stopping_config = config[\"eval\"].get(\"early_stopping\")\n    if early_stopping_config is not None:\n      early_stopping = EarlyStopping(\n          metric=early_stopping_config.get(\"metric\", \"loss\"),\n          min_improvement=early_stopping_config.get(\"min_improvement\", 0),\n          steps=early_stopping_config[\"steps\"])\n    else:\n      early_stopping = None\n    return cls(\n        model,\n        features_file or config[\"data\"][\"eval_features_file\"],\n        labels_file or config[\"data\"].get(\"eval_labels_file\"),\n        config[\"eval\"][\"batch_size\"],\n        scorers=scorers,\n        save_predictions=config[\"eval\"].get(\"save_eval_predictions\", False),\n        early_stopping=early_stopping,\n        eval_dir=os.path.join(config[\"model_dir\"], \"eval\"))\n\n  @property\n  def metrics_name(self):\n    \"\"\"The name of the metrics returned by this evaluator.\"\"\"\n    return self._metrics_name\n\n  @property\n  def metrics_history(self):\n    \"\"\"The history of metrics result per evaluation step.\"\"\"\n    return self._metrics_history\n\n  def should_stop(self):\n    \"\"\"Returns ``True`` if early stopping conditions are met.\"\"\"\n    if self._early_stopping is None:\n      return False\n    target_metric = self._early_stopping.metric\n    higher_is_better = None\n    # Look if target_metric is produced by a scorer as they define the scores order.\n    for scorer in self._scorers:\n      if target_metric in scorer.scores_name:\n        higher_is_better = scorer.higher_is_better()\n        break\n    if higher_is_better is None:\n      # TODO: the condition below is not always true, find a way to set it\n      # correctly for Keras metrics.\n      higher_is_better = target_metric != \"loss\"\n    metrics = [values[target_metric] for _, values in self._metrics_history]\n    should_stop = early_stop(\n        metrics,\n        self._early_stopping.steps,\n        min_improvement=self._early_stopping.min_improvement,\n        higher_is_better=higher_is_better)\n    if should_stop:\n      tf.get_logger().warning(\n          \"Evaluation metric '%s' did not improve more than %f in the last %d evaluations\",\n          target_metric,\n          self._early_stopping.min_improvement,\n          self._early_stopping.steps)\n    return should_stop\n\n  def __call__(self, step):\n    \"\"\"Runs the evaluator.\n\n    Args:\n      step: The current training step.\n\n    Returns:\n      A dictionary of evaluation metrics.\n    \"\"\"\n    tf.get_logger().info(\"Running evaluation for step %d\", step)\n    output_file = None\n    output_path = None\n    if self._save_predictions:\n      output_path = os.path.join(self._eval_dir, \"predictions.txt.%d\" % step)\n      output_file = tf.io.gfile.GFile(output_path, \"w\")\n\n    loss_num = 0\n    loss_den = 0\n    metrics = self._model.get_metrics()\n    for loss, predictions, target in self._eval():  # pylint: disable=no-value-for-parameter\n      if isinstance(loss, tuple):\n        loss_num += loss[0]\n        loss_den += loss[1]\n      else:\n        loss_num += loss\n        loss_den += 1\n      if metrics:\n        self._model.update_metrics(metrics, predictions, target)\n      if output_file is not None:\n        predictions = {k:v.numpy() for k, v in six.iteritems(predictions)}\n        for prediction in misc.extract_batches(predictions):\n          self._model.print_prediction(prediction, stream=output_file)\n    if loss_den == 0:\n      raise RuntimeError(\"No examples were evaluated\")\n    loss = loss_num / loss_den\n\n    results = dict(loss=loss)\n    if metrics:\n      for name, metric in six.iteritems(metrics):\n        results[name] = metric.result()\n    if self._save_predictions:\n      tf.get_logger().info(\"Evaluation predictions saved to %s\", output_path)\n      output_file.close()\n      for scorer in self._scorers:\n        score = scorer(self._labels_file, output_path)\n        if isinstance(score, dict):\n          results.update(score)\n        else:\n          results[scorer.name] = score\n\n    return self._record_results(step, results)\n\n  def _record_results(self, step, results):\n    for name, value in six.iteritems(results):\n      if isinstance(value, tf.Tensor):\n        results[name] = value.numpy()\n    # Clear history for steps that are greater than step.\n    while self._metrics_history and self._metrics_history[-1][0] > step:\n      self._metrics_history.pop()\n    self._metrics_history.append((step, dict(results)))\n    tf.get_logger().info(\n        \"Evaluation result for step %d: %s\",\n        step,\n        \" ; \".join(\"%s = %f\" % (k, v) for k, v in six.iteritems(results)))\n    with self._summary_writer.as_default():\n      for key, value in six.iteritems(results):\n        tf.summary.scalar(\"%s/%s\" % (_SUMMARIES_SCOPE, key), value, step=step)\n      self._summary_writer.flush()\n    return results\n\n\ndef early_stop(metrics, steps, min_improvement=0, higher_is_better=False):\n  \"\"\"Early stopping condition.\n\n  Args:\n    metrics: A list of metric values.\n    steps: Consider the improvement over this many steps.\n    min_improvement: Continue if the metric improved less than this value:\n    higher_is_better: Whether a higher value is better for this metric.\n\n  Returns:\n    A boolean.\n  \"\"\"\n  if len(metrics) < steps + 1:\n    return False\n\n  def _did_improve(ref, new):\n    # Returns True if new is improving on ref.\n    if higher_is_better:\n      return new > ref + min_improvement\n    else:\n      return new < ref - min_improvement\n\n  samples = metrics[-steps - 1:]\n\n  # First check if the boundaries show an improvement.\n  if _did_improve(samples[0], samples[-1]):\n    return False\n\n  # If not, only early stop if each successive evaluation did not improve.\n  for metric, next_metric in zip(samples[:-1], samples[1:]):\n    if _did_improve(metric, next_metric):\n      return False\n  return True\n", "path": "opennmt/evaluation.py"}], "after_files": [{"content": "\"\"\"Evaluation related classes and functions.\"\"\"\n\nimport collections\nimport os\nimport six\n\nimport tensorflow as tf\n\nfrom opennmt.data import dataset as dataset_lib\nfrom opennmt.utils import misc\nfrom opennmt.utils import scorers as scorers_lib\n\n\n_SUMMARIES_SCOPE = \"metrics\"\n\n\nclass EarlyStopping(\n    collections.namedtuple(\"EarlyStopping\",\n                           (\"metric\", \"min_improvement\", \"steps\"))):\n  \"\"\"Conditions for early stopping.\"\"\"\n\n\nclass Evaluator(object):\n  \"\"\"Model evaluator.\"\"\"\n\n  def __init__(self,\n               model,\n               features_file,\n               labels_file,\n               batch_size,\n               scorers=None,\n               save_predictions=False,\n               early_stopping=None,\n               eval_dir=None):\n    \"\"\"Initializes the evaluator.\n\n    Args:\n      model: A :class:`opennmt.models.model.Model` to evaluate.\n      features_file: Path to the evaluation features.\n      labels_file: Path to the evaluation labels.\n      batch_size: The evaluation batch size.\n      scorers: A list of scorers, callables taking the path to the reference and\n        the hypothesis and return one or more scores.\n      save_predictions: Save evaluation predictions to a file. This is ``True``\n        when :obj:`external_evaluator` is set.\n      early_stopping: An ``EarlyStopping`` instance.\n      eval_dir: Directory where predictions can be saved.\n\n    Raises:\n      ValueError: If predictions should be saved but the model is not compatible.\n      ValueError: If predictions should be saved but :obj:`eval_dir` is ``None``.\n      ValueError: If the :obj:`early_stopping` configuration is invalid.\n    \"\"\"\n    if scorers is None:\n      scorers = []\n    if scorers:\n      save_predictions = True\n    if save_predictions:\n      if model.unsupervised:\n        raise ValueError(\"This model does not support saving evaluation predictions\")\n      if eval_dir is None:\n        raise ValueError(\"Saving evaluation predictions requires eval_dir to be set\")\n      if not tf.io.gfile.exists(eval_dir):\n        tf.io.gfile.makedirs(eval_dir)\n    self._model = model\n    self._labels_file = labels_file\n    self._save_predictions = save_predictions\n    self._scorers = scorers\n    self._eval_dir = eval_dir\n    self._metrics_history = []\n    if eval_dir is not None:\n      self._summary_writer = tf.summary.create_file_writer(eval_dir)\n      summaries = misc.read_summaries(eval_dir)\n      for step, values in summaries:\n        metrics = misc.extract_prefixed_keys(values, _SUMMARIES_SCOPE + \"/\")\n        self._metrics_history.append((step, metrics))\n    else:\n      self._summary_writer = tf.summary.create_noop_writer()\n    dataset = model.examples_inputter.make_evaluation_dataset(\n        features_file,\n        labels_file,\n        batch_size,\n        num_threads=1,\n        prefetch_buffer_size=1)\n\n    @dataset_lib.function_on_next(dataset)\n    def _eval(next_fn):\n      source, target = next_fn()\n      outputs, predictions = model(source, labels=target)\n      loss = model.compute_loss(outputs, target, training=False)\n      return loss, predictions, target\n\n    self._eval = _eval\n\n    self._metrics_name = {\"loss\", \"perplexity\"}\n    for scorer in self._scorers:\n      self._metrics_name.update(scorer.scores_name)\n    model_metrics = self._model.get_metrics()\n    if model_metrics:\n      self._metrics_name.update(set(six.iterkeys(model_metrics)))\n\n    if early_stopping is not None:\n      if early_stopping.metric not in self._metrics_name:\n        raise ValueError(\"Invalid early stopping metric '%s', expected one in %s\" % (\n            early_stopping.metric, str(self._metrics_name)))\n      if early_stopping.steps <= 0:\n        raise ValueError(\"Early stopping steps should greater than 0\")\n    self._early_stopping = early_stopping\n\n  @classmethod\n  def from_config(cls, model, config, features_file=None, labels_file=None):\n    \"\"\"Creates an evaluator from the configuration.\n\n    Args:\n      model: A :class:`opennmt.models.model.Model` to evaluate.\n      config: The global user configuration.\n      features_file: Optional input features file to evaluate. If not set, will\n        load ``eval_features_file`` from the data configuration.\n      labels_file: Optional output labels file to evaluate. If not set, will load\n        ``eval_labels_file`` from the data configuration.\n\n    Returns:\n      A :class:`opennmt.evaluation.Evaluator` instance.\n\n    Raises:\n      ValueError: if one of :obj:`features_file` and :obj:`labels_file` is set\n        but not the other.\n    \"\"\"\n    if (features_file is None) != (labels_file is None):\n      raise ValueError(\"features_file and labels_file should be both set for evaluation\")\n    scorers = config[\"eval\"].get(\"external_evaluators\")\n    if scorers is not None:\n      scorers = scorers_lib.make_scorers(scorers)\n    early_stopping_config = config[\"eval\"].get(\"early_stopping\")\n    if early_stopping_config is not None:\n      early_stopping = EarlyStopping(\n          metric=early_stopping_config.get(\"metric\", \"loss\"),\n          min_improvement=early_stopping_config.get(\"min_improvement\", 0),\n          steps=early_stopping_config[\"steps\"])\n    else:\n      early_stopping = None\n    return cls(\n        model,\n        features_file or config[\"data\"][\"eval_features_file\"],\n        labels_file or config[\"data\"].get(\"eval_labels_file\"),\n        config[\"eval\"][\"batch_size\"],\n        scorers=scorers,\n        save_predictions=config[\"eval\"].get(\"save_eval_predictions\", False),\n        early_stopping=early_stopping,\n        eval_dir=os.path.join(config[\"model_dir\"], \"eval\"))\n\n  @property\n  def metrics_name(self):\n    \"\"\"The name of the metrics returned by this evaluator.\"\"\"\n    return self._metrics_name\n\n  @property\n  def metrics_history(self):\n    \"\"\"The history of metrics result per evaluation step.\"\"\"\n    return self._metrics_history\n\n  def should_stop(self):\n    \"\"\"Returns ``True`` if early stopping conditions are met.\"\"\"\n    if self._early_stopping is None:\n      return False\n    target_metric = self._early_stopping.metric\n    higher_is_better = None\n    # Look if target_metric is produced by a scorer as they define the scores order.\n    for scorer in self._scorers:\n      if target_metric in scorer.scores_name:\n        higher_is_better = scorer.higher_is_better()\n        break\n    if higher_is_better is None:\n      # TODO: the condition below is not always true, find a way to set it\n      # correctly for Keras metrics.\n      higher_is_better = target_metric not in (\"loss\", \"perplexity\")\n    metrics = [values[target_metric] for _, values in self._metrics_history]\n    should_stop = early_stop(\n        metrics,\n        self._early_stopping.steps,\n        min_improvement=self._early_stopping.min_improvement,\n        higher_is_better=higher_is_better)\n    if should_stop:\n      tf.get_logger().warning(\n          \"Evaluation metric '%s' did not improve more than %f in the last %d evaluations\",\n          target_metric,\n          self._early_stopping.min_improvement,\n          self._early_stopping.steps)\n    return should_stop\n\n  def __call__(self, step):\n    \"\"\"Runs the evaluator.\n\n    Args:\n      step: The current training step.\n\n    Returns:\n      A dictionary of evaluation metrics.\n    \"\"\"\n    tf.get_logger().info(\"Running evaluation for step %d\", step)\n    output_file = None\n    output_path = None\n    if self._save_predictions:\n      output_path = os.path.join(self._eval_dir, \"predictions.txt.%d\" % step)\n      output_file = tf.io.gfile.GFile(output_path, \"w\")\n\n    loss_num = 0\n    loss_den = 0\n    metrics = self._model.get_metrics()\n    for loss, predictions, target in self._eval():  # pylint: disable=no-value-for-parameter\n      if isinstance(loss, tuple):\n        loss_num += loss[0]\n        loss_den += loss[1]\n      else:\n        loss_num += loss\n        loss_den += 1\n      if metrics:\n        self._model.update_metrics(metrics, predictions, target)\n      if output_file is not None:\n        predictions = {k:v.numpy() for k, v in six.iteritems(predictions)}\n        for prediction in misc.extract_batches(predictions):\n          self._model.print_prediction(prediction, stream=output_file)\n    if loss_den == 0:\n      raise RuntimeError(\"No examples were evaluated\")\n    loss = loss_num / loss_den\n\n    results = dict(loss=loss, perplexity=tf.math.exp(loss))\n    if metrics:\n      for name, metric in six.iteritems(metrics):\n        results[name] = metric.result()\n    if self._save_predictions:\n      tf.get_logger().info(\"Evaluation predictions saved to %s\", output_path)\n      output_file.close()\n      for scorer in self._scorers:\n        score = scorer(self._labels_file, output_path)\n        if isinstance(score, dict):\n          results.update(score)\n        else:\n          results[scorer.name] = score\n\n    return self._record_results(step, results)\n\n  def _record_results(self, step, results):\n    for name, value in six.iteritems(results):\n      if isinstance(value, tf.Tensor):\n        results[name] = value.numpy()\n    # Clear history for steps that are greater than step.\n    while self._metrics_history and self._metrics_history[-1][0] > step:\n      self._metrics_history.pop()\n    self._metrics_history.append((step, dict(results)))\n    tf.get_logger().info(\n        \"Evaluation result for step %d: %s\",\n        step,\n        \" ; \".join(\"%s = %f\" % (k, v) for k, v in six.iteritems(results)))\n    with self._summary_writer.as_default():\n      for key, value in six.iteritems(results):\n        tf.summary.scalar(\"%s/%s\" % (_SUMMARIES_SCOPE, key), value, step=step)\n      self._summary_writer.flush()\n    return results\n\n\ndef early_stop(metrics, steps, min_improvement=0, higher_is_better=False):\n  \"\"\"Early stopping condition.\n\n  Args:\n    metrics: A list of metric values.\n    steps: Consider the improvement over this many steps.\n    min_improvement: Continue if the metric improved less than this value:\n    higher_is_better: Whether a higher value is better for this metric.\n\n  Returns:\n    A boolean.\n  \"\"\"\n  if len(metrics) < steps + 1:\n    return False\n\n  def _did_improve(ref, new):\n    # Returns True if new is improving on ref.\n    if higher_is_better:\n      return new > ref + min_improvement\n    else:\n      return new < ref - min_improvement\n\n  samples = metrics[-steps - 1:]\n\n  # First check if the boundaries show an improvement.\n  if _did_improve(samples[0], samples[-1]):\n    return False\n\n  # If not, only early stop if each successive evaluation did not improve.\n  for metric, next_metric in zip(samples[:-1], samples[1:]):\n    if _did_improve(metric, next_metric):\n      return False\n  return True\n", "path": "opennmt/evaluation.py"}]}