We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
problem for compute the "real" BLEU 
it seems has some small problems.

Traceback (most recent call last):
File "/usr/local/bin/t2t-bleu", line 6, in
exec(compile(open(file).read(), file, 'exec'))
File "/workspace/tf_project/tensor2tensor/tensor2tensor/bin/t2t-bleu", line 137, in
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 48, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "/workspace/tf_project/tensor2tensor/tensor2tensor/bin/t2t-bleu", line 80, in main
bleu = 100 * bleu_hook.bleu_wrapper(FLAGS.reference, FLAGS.translation, case_sensitive=False)
File "/workspace/tf_project/tensor2tensor/tensor2tensor/utils/bleu_hook.py", line 202, in bleu_wrapper
ref_tokens = [bleu_tokenize(x) for x in ref_lines]
File "/workspace/tf_project/tensor2tensor/tensor2tensor/utils/bleu_hook.py", line 187, in bleu_tokenize
string = UnicodeRegex.nondigit_punct_re.sub(r'\1 \2 ', string)
AttributeError: type object 'UnicodeRegex' has no attribute 'nondigit_punct_re'

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `tensor2tensor/utils/bleu_hook.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2017 The Tensor2Tensor Authors.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 
16 """BLEU metric util used during eval for MT."""
17 from __future__ import absolute_import
18 from __future__ import division
19 from __future__ import print_function
20 
21 import collections
22 import math
23 import os
24 import re
25 import sys
26 import time
27 import unicodedata
28 from collections import namedtuple
29 
30 # Dependency imports
31 
32 import numpy as np
33 import six
34 # pylint: disable=redefined-builtin
35 from six.moves import xrange
36 from six.moves import zip
37 # pylint: enable=redefined-builtin
38 
39 import tensorflow as tf
40 
41 
42 def _get_ngrams(segment, max_order):
43   """Extracts all n-grams upto a given maximum order from an input segment.
44 
45   Args:
46     segment: text segment from which n-grams will be extracted.
47     max_order: maximum length in tokens of the n-grams returned by this
48         methods.
49 
50   Returns:
51     The Counter containing all n-grams upto max_order in segment
52     with a count of how many times each n-gram occurred.
53   """
54   ngram_counts = collections.Counter()
55   for order in xrange(1, max_order + 1):
56     for i in xrange(0, len(segment) - order + 1):
57       ngram = tuple(segment[i:i + order])
58       ngram_counts[ngram] += 1
59   return ngram_counts
60 
61 
62 def compute_bleu(reference_corpus,
63                  translation_corpus,
64                  max_order=4,
65                  use_bp=True):
66   """Computes BLEU score of translated segments against one or more references.
67 
68   Args:
69     reference_corpus: list of references for each translation. Each
70         reference should be tokenized into a list of tokens.
71     translation_corpus: list of translations to score. Each translation
72         should be tokenized into a list of tokens.
73     max_order: Maximum n-gram order to use when computing BLEU score.
74     use_bp: boolean, whether to apply brevity penalty.
75 
76   Returns:
77     BLEU score.
78   """
79   reference_length = 0
80   translation_length = 0
81   bp = 1.0
82   geo_mean = 0
83 
84   matches_by_order = [0] * max_order
85   possible_matches_by_order = [0] * max_order
86   precisions = []
87 
88   for (references, translations) in zip(reference_corpus, translation_corpus):
89     reference_length += len(references)
90     translation_length += len(translations)
91     ref_ngram_counts = _get_ngrams(references, max_order)
92     translation_ngram_counts = _get_ngrams(translations, max_order)
93 
94     overlap = dict((ngram,
95                     min(count, translation_ngram_counts[ngram]))
96                    for ngram, count in ref_ngram_counts.items())
97 
98     for ngram in overlap:
99       matches_by_order[len(ngram) - 1] += overlap[ngram]
100     for ngram in translation_ngram_counts:
101       possible_matches_by_order[len(ngram)-1] += translation_ngram_counts[ngram]
102   precisions = [0] * max_order
103   smooth = 1.0
104   for i in xrange(0, max_order):
105     if possible_matches_by_order[i] > 0:
106       precisions[i] = matches_by_order[i] / possible_matches_by_order[i]
107       if matches_by_order[i] > 0:
108         precisions[i] = matches_by_order[i] / possible_matches_by_order[i]
109       else:
110         smooth *= 2
111         precisions[i] = 1.0 / (smooth * possible_matches_by_order[i])
112     else:
113       precisions[i] = 0.0
114 
115   if max(precisions) > 0:
116     p_log_sum = sum(math.log(p) for p in precisions if p)
117     geo_mean = math.exp(p_log_sum/max_order)
118 
119   if use_bp:
120     ratio = translation_length / reference_length
121     bp = math.exp(1 - 1. / ratio) if ratio < 1.0 else 1.0
122   bleu = geo_mean * bp
123   return np.float32(bleu)
124 
125 
126 def bleu_score(predictions, labels, **unused_kwargs):
127   """BLEU score computation between labels and predictions.
128 
129   An approximate BLEU scoring method since we do not glue word pieces or
130   decode the ids and tokenize the output. By default, we use ngram order of 4
131   and use brevity penalty. Also, this does not have beam search.
132 
133   Args:
134     predictions: tensor, model predicitons
135     labels: tensor, gold output.
136 
137   Returns:
138     bleu: int, approx bleu score
139   """
140   outputs = tf.to_int32(tf.argmax(predictions, axis=-1))
141   # Convert the outputs and labels to a [batch_size, input_length] tensor.
142   outputs = tf.squeeze(outputs, axis=[-1, -2])
143   labels = tf.squeeze(labels, axis=[-1, -2])
144 
145   bleu = tf.py_func(compute_bleu, (labels, outputs), tf.float32)
146   return bleu, tf.constant(1.0)
147 
148 
149 class UnicodeRegex(object):
150   """Ad-hoc hack to recognize all punctuation and symbols."""
151 
152   def __init__(self):
153     def _property_chars(prefix):
154       return ''.join(six.unichr(x) for x in range(sys.maxunicode)
155                      if unicodedata.category(six.unichr(x)).startswith(prefix))
156     punctuation = self._property_chars('P')
157     self.nondigit_punct_re = re.compile(r'([^\d])([' + punctuation + r'])')
158     self.punct_nondigit_re = re.compile(r'([' + punctuation + r'])([^\d])')
159     self.symbol_re = re.compile('([' + _property_chars('S') + '])')
160 
161 
162 def bleu_tokenize(string):
163   r"""Tokenize a string following the official BLEU implementation.
164 
165   See https://github.com/moses-smt/mosesdecoder/"
166            "blob/master/scripts/generic/mteval-v14.pl#L954-L983
167   In our case, the input string is expected to be just one line
168   and no HTML entities de-escaping is needed.
169   So we just tokenize on punctuation and symbols,
170   except when a punctuation is preceded and followed by a digit
171   (e.g. a comma/dot as a thousand/decimal separator).
172 
173   Note that a numer (e.g. a year) followed by a dot at the end of sentence
174   is NOT tokenized,
175   i.e. the dot stays with the number because `s/(\p{P})(\P{N})/ $1 $2/g`
176   does not match this case (unless we add a space after each sentence).
177   However, this error is already in the original mteval-v14.pl
178   and we want to be consistent with it.
179 
180   Args:
181     string: the input string
182 
183   Returns:
184     a list of tokens
185   """
186   string = UnicodeRegex.nondigit_punct_re.sub(r'\1 \2 ', string)
187   string = UnicodeRegex.punct_nondigit_re.sub(r' \1 \2', string)
188   string = UnicodeRegex.symbol_re.sub(r' \1 ', string)
189   return string.split()
190 
191 
192 def bleu_wrapper(ref_filename, hyp_filename, case_sensitive=False):
193   """Compute BLEU for two files (reference and hypothesis translation)."""
194   ref_lines = open(ref_filename).read().splitlines()
195   hyp_lines = open(hyp_filename).read().splitlines()
196   assert len(ref_lines) == len(hyp_lines)
197   if not case_sensitive:
198     ref_lines = [x.lower() for x in ref_lines]
199     hyp_lines = [x.lower() for x in hyp_lines]
200   ref_tokens = [bleu_tokenize(x) for x in ref_lines]
201   hyp_tokens = [bleu_tokenize(x) for x in hyp_lines]
202   return compute_bleu(ref_tokens, hyp_tokens)
203 
204 
205 StepFile = namedtuple('StepFile', 'filename mtime ctime steps')
206 
207 
208 def _read_stepfiles_list(path_prefix, path_suffix='.index', min_steps=0):
209   stepfiles = []
210   for filename in tf.gfile.Glob(path_prefix + '*-[0-9]*' + path_suffix):
211     basename = filename[:-len(path_suffix)] if len(path_suffix) else filename
212     try:
213       steps = int(basename.rsplit('-')[-1])
214     except ValueError:  # The -[0-9]* part is not an integer.
215       continue
216     if steps < min_steps:
217       continue
218     if not os.path.exists(filename):
219       tf.logging.info(filename + " was deleted, so skipping it")
220       continue
221     stepfiles.append(StepFile(basename, os.path.getmtime(filename),
222                               os.path.getctime(filename), steps))
223   return sorted(stepfiles, key=lambda x: -x.steps)
224 
225 
226 def stepfiles_iterator(path_prefix, wait_minutes=0, min_steps=0,
227                        path_suffix='.index', sleep_sec=10):
228   """Continuously yield new files with steps in filename as they appear.
229 
230   This is useful for checkpoint files or other files whose names differ just in an interger
231   marking the number of steps and match the wildcard path_prefix + '*-[0-9]*' + path_suffix.
232   Unlike `tf.contrib.training.checkpoints_iterator`, this
233   implementation always starts from the oldest files
234   (and it cannot miss any file). Note that the oldest checkpoint
235   may be deleted anytime by Tensorflow (if set up so). It is up to the user
236   to check that the files returned by this generator actually exist.
237   Args:
238     path_prefix: The directory + possible common filename prefix to the files.
239     path_suffix: Common filename suffix (after steps), including possible extension dot.
240     wait_minutes: The maximum amount of minutes to wait between files.
241     min_steps: Skip files with lower global step.
242     sleep_sec: How often to check for new files.
243   Yields:
244     named tuples (filename, mtime, ctime, steps) of the files as they arrive.
245   """
246   # Wildcard D*-[0-9]* does not match D/x-1, so if D is a directory let path_prefix='D/'.
247   if not path_prefix.endswith(os.sep) and os.path.isdir(path_prefix):
248     path_prefix += os.sep
249   stepfiles = _read_stepfiles_list(path_prefix, path_suffix, min_steps)
250   tf.logging.info("Found %d files with steps: %s"
251                   % (len(stepfiles), ", ".join(str(x.steps) for x in reversed(stepfiles))))
252   exit_time = time.time() + wait_minutes * 60
253   while True:
254     if not stepfiles and wait_minutes:
255       tf.logging.info('Waiting till %s if a new file matching %s*-[0-9]*%s appears'
256                       % (time.asctime(time.localtime(exit_time)), path_prefix, path_suffix))
257       while True:
258         stepfiles = _read_stepfiles_list(path_prefix, path_suffix, min_steps)
259         if stepfiles or time.time() > exit_time:
260           break
261         time.sleep(sleep_sec)
262     if not stepfiles:
263       return
264 
265     stepfile = stepfiles.pop()
266     exit_time, min_steps = stepfile.ctime + wait_minutes * 60, stepfile.steps + 1
267     yield stepfile
268
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/tensor2tensor/utils/bleu_hook.py b/tensor2tensor/utils/bleu_hook.py
--- a/tensor2tensor/utils/bleu_hook.py
+++ b/tensor2tensor/utils/bleu_hook.py
@@ -153,7 +153,7 @@
     def _property_chars(prefix):
       return ''.join(six.unichr(x) for x in range(sys.maxunicode)
                      if unicodedata.category(six.unichr(x)).startswith(prefix))
-    punctuation = self._property_chars('P')
+    punctuation = _property_chars('P')
     self.nondigit_punct_re = re.compile(r'([^\d])([' + punctuation + r'])')
     self.punct_nondigit_re = re.compile(r'([' + punctuation + r'])([^\d])')
     self.symbol_re = re.compile('([' + _property_chars('S') + '])')
@@ -183,9 +183,10 @@
   Returns:
     a list of tokens
   """
-  string = UnicodeRegex.nondigit_punct_re.sub(r'\1 \2 ', string)
-  string = UnicodeRegex.punct_nondigit_re.sub(r' \1 \2', string)
-  string = UnicodeRegex.symbol_re.sub(r' \1 ', string)
+  uregex = UnicodeRegex()
+  string = uregex.nondigit_punct_re.sub(r'\1 \2 ', string)
+  string = uregex.punct_nondigit_re.sub(r' \1 \2', string)
+  string = uregex.symbol_re.sub(r' \1 ', string)
   return string.split()

{"golden_diff": "diff --git a/tensor2tensor/utils/bleu_hook.py b/tensor2tensor/utils/bleu_hook.py\n--- a/tensor2tensor/utils/bleu_hook.py\n+++ b/tensor2tensor/utils/bleu_hook.py\n@@ -153,7 +153,7 @@\n     def _property_chars(prefix):\n       return ''.join(six.unichr(x) for x in range(sys.maxunicode)\n                      if unicodedata.category(six.unichr(x)).startswith(prefix))\n-    punctuation = self._property_chars('P')\n+    punctuation = _property_chars('P')\n     self.nondigit_punct_re = re.compile(r'([^\\d])([' + punctuation + r'])')\n     self.punct_nondigit_re = re.compile(r'([' + punctuation + r'])([^\\d])')\n     self.symbol_re = re.compile('([' + _property_chars('S') + '])')\n@@ -183,9 +183,10 @@\n   Returns:\n     a list of tokens\n   \"\"\"\n-  string = UnicodeRegex.nondigit_punct_re.sub(r'\\1 \\2 ', string)\n-  string = UnicodeRegex.punct_nondigit_re.sub(r' \\1 \\2', string)\n-  string = UnicodeRegex.symbol_re.sub(r' \\1 ', string)\n+  uregex = UnicodeRegex()\n+  string = uregex.nondigit_punct_re.sub(r'\\1 \\2 ', string)\n+  string = uregex.punct_nondigit_re.sub(r' \\1 \\2', string)\n+  string = uregex.symbol_re.sub(r' \\1 ', string)\n   return string.split()\n", "issue": "problem for compute the \"real\" BLEU \nit seems has some small problems.\r\n\r\nTraceback (most recent call last):\r\nFile \"/usr/local/bin/t2t-bleu\", line 6, in\r\nexec(compile(open(file).read(), file, 'exec'))\r\nFile \"/workspace/tf_project/tensor2tensor/tensor2tensor/bin/t2t-bleu\", line 137, in\r\ntf.app.run()\r\nFile \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py\", line 48, in run\r\n_sys.exit(main(_sys.argv[:1] + flags_passthrough))\r\nFile \"/workspace/tf_project/tensor2tensor/tensor2tensor/bin/t2t-bleu\", line 80, in main\r\nbleu = 100 * bleu_hook.bleu_wrapper(FLAGS.reference, FLAGS.translation, case_sensitive=False)\r\nFile \"/workspace/tf_project/tensor2tensor/tensor2tensor/utils/bleu_hook.py\", line 202, in bleu_wrapper\r\nref_tokens = [bleu_tokenize(x) for x in ref_lines]\r\nFile \"/workspace/tf_project/tensor2tensor/tensor2tensor/utils/bleu_hook.py\", line 187, in bleu_tokenize\r\nstring = UnicodeRegex.nondigit_punct_re.sub(r'\\1 \\2 ', string)\r\nAttributeError: type object 'UnicodeRegex' has no attribute 'nondigit_punct_re'\n", "before_files": [{"content": "# coding=utf-8\n# Copyright 2017 The Tensor2Tensor Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"BLEU metric util used during eval for MT.\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport collections\nimport math\nimport os\nimport re\nimport sys\nimport time\nimport unicodedata\nfrom collections import namedtuple\n\n# Dependency imports\n\nimport numpy as np\nimport six\n# pylint: disable=redefined-builtin\nfrom six.moves import xrange\nfrom six.moves import zip\n# pylint: enable=redefined-builtin\n\nimport tensorflow as tf\n\n\ndef _get_ngrams(segment, max_order):\n  \"\"\"Extracts all n-grams upto a given maximum order from an input segment.\n\n  Args:\n    segment: text segment from which n-grams will be extracted.\n    max_order: maximum length in tokens of the n-grams returned by this\n        methods.\n\n  Returns:\n    The Counter containing all n-grams upto max_order in segment\n    with a count of how many times each n-gram occurred.\n  \"\"\"\n  ngram_counts = collections.Counter()\n  for order in xrange(1, max_order + 1):\n    for i in xrange(0, len(segment) - order + 1):\n      ngram = tuple(segment[i:i + order])\n      ngram_counts[ngram] += 1\n  return ngram_counts\n\n\ndef compute_bleu(reference_corpus,\n                 translation_corpus,\n                 max_order=4,\n                 use_bp=True):\n  \"\"\"Computes BLEU score of translated segments against one or more references.\n\n  Args:\n    reference_corpus: list of references for each translation. Each\n        reference should be tokenized into a list of tokens.\n    translation_corpus: list of translations to score. Each translation\n        should be tokenized into a list of tokens.\n    max_order: Maximum n-gram order to use when computing BLEU score.\n    use_bp: boolean, whether to apply brevity penalty.\n\n  Returns:\n    BLEU score.\n  \"\"\"\n  reference_length = 0\n  translation_length = 0\n  bp = 1.0\n  geo_mean = 0\n\n  matches_by_order = [0] * max_order\n  possible_matches_by_order = [0] * max_order\n  precisions = []\n\n  for (references, translations) in zip(reference_corpus, translation_corpus):\n    reference_length += len(references)\n    translation_length += len(translations)\n    ref_ngram_counts = _get_ngrams(references, max_order)\n    translation_ngram_counts = _get_ngrams(translations, max_order)\n\n    overlap = dict((ngram,\n                    min(count, translation_ngram_counts[ngram]))\n                   for ngram, count in ref_ngram_counts.items())\n\n    for ngram in overlap:\n      matches_by_order[len(ngram) - 1] += overlap[ngram]\n    for ngram in translation_ngram_counts:\n      possible_matches_by_order[len(ngram)-1] += translation_ngram_counts[ngram]\n  precisions = [0] * max_order\n  smooth = 1.0\n  for i in xrange(0, max_order):\n    if possible_matches_by_order[i] > 0:\n      precisions[i] = matches_by_order[i] / possible_matches_by_order[i]\n      if matches_by_order[i] > 0:\n        precisions[i] = matches_by_order[i] / possible_matches_by_order[i]\n      else:\n        smooth *= 2\n        precisions[i] = 1.0 / (smooth * possible_matches_by_order[i])\n    else:\n      precisions[i] = 0.0\n\n  if max(precisions) > 0:\n    p_log_sum = sum(math.log(p) for p in precisions if p)\n    geo_mean = math.exp(p_log_sum/max_order)\n\n  if use_bp:\n    ratio = translation_length / reference_length\n    bp = math.exp(1 - 1. / ratio) if ratio < 1.0 else 1.0\n  bleu = geo_mean * bp\n  return np.float32(bleu)\n\n\ndef bleu_score(predictions, labels, **unused_kwargs):\n  \"\"\"BLEU score computation between labels and predictions.\n\n  An approximate BLEU scoring method since we do not glue word pieces or\n  decode the ids and tokenize the output. By default, we use ngram order of 4\n  and use brevity penalty. Also, this does not have beam search.\n\n  Args:\n    predictions: tensor, model predicitons\n    labels: tensor, gold output.\n\n  Returns:\n    bleu: int, approx bleu score\n  \"\"\"\n  outputs = tf.to_int32(tf.argmax(predictions, axis=-1))\n  # Convert the outputs and labels to a [batch_size, input_length] tensor.\n  outputs = tf.squeeze(outputs, axis=[-1, -2])\n  labels = tf.squeeze(labels, axis=[-1, -2])\n\n  bleu = tf.py_func(compute_bleu, (labels, outputs), tf.float32)\n  return bleu, tf.constant(1.0)\n\n\nclass UnicodeRegex(object):\n  \"\"\"Ad-hoc hack to recognize all punctuation and symbols.\"\"\"\n\n  def __init__(self):\n    def _property_chars(prefix):\n      return ''.join(six.unichr(x) for x in range(sys.maxunicode)\n                     if unicodedata.category(six.unichr(x)).startswith(prefix))\n    punctuation = self._property_chars('P')\n    self.nondigit_punct_re = re.compile(r'([^\\d])([' + punctuation + r'])')\n    self.punct_nondigit_re = re.compile(r'([' + punctuation + r'])([^\\d])')\n    self.symbol_re = re.compile('([' + _property_chars('S') + '])')\n\n\ndef bleu_tokenize(string):\n  r\"\"\"Tokenize a string following the official BLEU implementation.\n\n  See https://github.com/moses-smt/mosesdecoder/\"\n           \"blob/master/scripts/generic/mteval-v14.pl#L954-L983\n  In our case, the input string is expected to be just one line\n  and no HTML entities de-escaping is needed.\n  So we just tokenize on punctuation and symbols,\n  except when a punctuation is preceded and followed by a digit\n  (e.g. a comma/dot as a thousand/decimal separator).\n\n  Note that a numer (e.g. a year) followed by a dot at the end of sentence\n  is NOT tokenized,\n  i.e. the dot stays with the number because `s/(\\p{P})(\\P{N})/ $1 $2/g`\n  does not match this case (unless we add a space after each sentence).\n  However, this error is already in the original mteval-v14.pl\n  and we want to be consistent with it.\n\n  Args:\n    string: the input string\n\n  Returns:\n    a list of tokens\n  \"\"\"\n  string = UnicodeRegex.nondigit_punct_re.sub(r'\\1 \\2 ', string)\n  string = UnicodeRegex.punct_nondigit_re.sub(r' \\1 \\2', string)\n  string = UnicodeRegex.symbol_re.sub(r' \\1 ', string)\n  return string.split()\n\n\ndef bleu_wrapper(ref_filename, hyp_filename, case_sensitive=False):\n  \"\"\"Compute BLEU for two files (reference and hypothesis translation).\"\"\"\n  ref_lines = open(ref_filename).read().splitlines()\n  hyp_lines = open(hyp_filename).read().splitlines()\n  assert len(ref_lines) == len(hyp_lines)\n  if not case_sensitive:\n    ref_lines = [x.lower() for x in ref_lines]\n    hyp_lines = [x.lower() for x in hyp_lines]\n  ref_tokens = [bleu_tokenize(x) for x in ref_lines]\n  hyp_tokens = [bleu_tokenize(x) for x in hyp_lines]\n  return compute_bleu(ref_tokens, hyp_tokens)\n\n\nStepFile = namedtuple('StepFile', 'filename mtime ctime steps')\n\n\ndef _read_stepfiles_list(path_prefix, path_suffix='.index', min_steps=0):\n  stepfiles = []\n  for filename in tf.gfile.Glob(path_prefix + '*-[0-9]*' + path_suffix):\n    basename = filename[:-len(path_suffix)] if len(path_suffix) else filename\n    try:\n      steps = int(basename.rsplit('-')[-1])\n    except ValueError:  # The -[0-9]* part is not an integer.\n      continue\n    if steps < min_steps:\n      continue\n    if not os.path.exists(filename):\n      tf.logging.info(filename + \" was deleted, so skipping it\")\n      continue\n    stepfiles.append(StepFile(basename, os.path.getmtime(filename),\n                              os.path.getctime(filename), steps))\n  return sorted(stepfiles, key=lambda x: -x.steps)\n\n\ndef stepfiles_iterator(path_prefix, wait_minutes=0, min_steps=0,\n                       path_suffix='.index', sleep_sec=10):\n  \"\"\"Continuously yield new files with steps in filename as they appear.\n\n  This is useful for checkpoint files or other files whose names differ just in an interger\n  marking the number of steps and match the wildcard path_prefix + '*-[0-9]*' + path_suffix.\n  Unlike `tf.contrib.training.checkpoints_iterator`, this\n  implementation always starts from the oldest files\n  (and it cannot miss any file). Note that the oldest checkpoint\n  may be deleted anytime by Tensorflow (if set up so). It is up to the user\n  to check that the files returned by this generator actually exist.\n  Args:\n    path_prefix: The directory + possible common filename prefix to the files.\n    path_suffix: Common filename suffix (after steps), including possible extension dot.\n    wait_minutes: The maximum amount of minutes to wait between files.\n    min_steps: Skip files with lower global step.\n    sleep_sec: How often to check for new files.\n  Yields:\n    named tuples (filename, mtime, ctime, steps) of the files as they arrive.\n  \"\"\"\n  # Wildcard D*-[0-9]* does not match D/x-1, so if D is a directory let path_prefix='D/'.\n  if not path_prefix.endswith(os.sep) and os.path.isdir(path_prefix):\n    path_prefix += os.sep\n  stepfiles = _read_stepfiles_list(path_prefix, path_suffix, min_steps)\n  tf.logging.info(\"Found %d files with steps: %s\"\n                  % (len(stepfiles), \", \".join(str(x.steps) for x in reversed(stepfiles))))\n  exit_time = time.time() + wait_minutes * 60\n  while True:\n    if not stepfiles and wait_minutes:\n      tf.logging.info('Waiting till %s if a new file matching %s*-[0-9]*%s appears'\n                      % (time.asctime(time.localtime(exit_time)), path_prefix, path_suffix))\n      while True:\n        stepfiles = _read_stepfiles_list(path_prefix, path_suffix, min_steps)\n        if stepfiles or time.time() > exit_time:\n          break\n        time.sleep(sleep_sec)\n    if not stepfiles:\n      return\n\n    stepfile = stepfiles.pop()\n    exit_time, min_steps = stepfile.ctime + wait_minutes * 60, stepfile.steps + 1\n    yield stepfile\n", "path": "tensor2tensor/utils/bleu_hook.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright 2017 The Tensor2Tensor Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"BLEU metric util used during eval for MT.\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport collections\nimport math\nimport os\nimport re\nimport sys\nimport time\nimport unicodedata\nfrom collections import namedtuple\n\n# Dependency imports\n\nimport numpy as np\nimport six\n# pylint: disable=redefined-builtin\nfrom six.moves import xrange\nfrom six.moves import zip\n# pylint: enable=redefined-builtin\n\nimport tensorflow as tf\n\n\ndef _get_ngrams(segment, max_order):\n  \"\"\"Extracts all n-grams upto a given maximum order from an input segment.\n\n  Args:\n    segment: text segment from which n-grams will be extracted.\n    max_order: maximum length in tokens of the n-grams returned by this\n        methods.\n\n  Returns:\n    The Counter containing all n-grams upto max_order in segment\n    with a count of how many times each n-gram occurred.\n  \"\"\"\n  ngram_counts = collections.Counter()\n  for order in xrange(1, max_order + 1):\n    for i in xrange(0, len(segment) - order + 1):\n      ngram = tuple(segment[i:i + order])\n      ngram_counts[ngram] += 1\n  return ngram_counts\n\n\ndef compute_bleu(reference_corpus,\n                 translation_corpus,\n                 max_order=4,\n                 use_bp=True):\n  \"\"\"Computes BLEU score of translated segments against one or more references.\n\n  Args:\n    reference_corpus: list of references for each translation. Each\n        reference should be tokenized into a list of tokens.\n    translation_corpus: list of translations to score. Each translation\n        should be tokenized into a list of tokens.\n    max_order: Maximum n-gram order to use when computing BLEU score.\n    use_bp: boolean, whether to apply brevity penalty.\n\n  Returns:\n    BLEU score.\n  \"\"\"\n  reference_length = 0\n  translation_length = 0\n  bp = 1.0\n  geo_mean = 0\n\n  matches_by_order = [0] * max_order\n  possible_matches_by_order = [0] * max_order\n  precisions = []\n\n  for (references, translations) in zip(reference_corpus, translation_corpus):\n    reference_length += len(references)\n    translation_length += len(translations)\n    ref_ngram_counts = _get_ngrams(references, max_order)\n    translation_ngram_counts = _get_ngrams(translations, max_order)\n\n    overlap = dict((ngram,\n                    min(count, translation_ngram_counts[ngram]))\n                   for ngram, count in ref_ngram_counts.items())\n\n    for ngram in overlap:\n      matches_by_order[len(ngram) - 1] += overlap[ngram]\n    for ngram in translation_ngram_counts:\n      possible_matches_by_order[len(ngram)-1] += translation_ngram_counts[ngram]\n  precisions = [0] * max_order\n  smooth = 1.0\n  for i in xrange(0, max_order):\n    if possible_matches_by_order[i] > 0:\n      precisions[i] = matches_by_order[i] / possible_matches_by_order[i]\n      if matches_by_order[i] > 0:\n        precisions[i] = matches_by_order[i] / possible_matches_by_order[i]\n      else:\n        smooth *= 2\n        precisions[i] = 1.0 / (smooth * possible_matches_by_order[i])\n    else:\n      precisions[i] = 0.0\n\n  if max(precisions) > 0:\n    p_log_sum = sum(math.log(p) for p in precisions if p)\n    geo_mean = math.exp(p_log_sum/max_order)\n\n  if use_bp:\n    ratio = translation_length / reference_length\n    bp = math.exp(1 - 1. / ratio) if ratio < 1.0 else 1.0\n  bleu = geo_mean * bp\n  return np.float32(bleu)\n\n\ndef bleu_score(predictions, labels, **unused_kwargs):\n  \"\"\"BLEU score computation between labels and predictions.\n\n  An approximate BLEU scoring method since we do not glue word pieces or\n  decode the ids and tokenize the output. By default, we use ngram order of 4\n  and use brevity penalty. Also, this does not have beam search.\n\n  Args:\n    predictions: tensor, model predicitons\n    labels: tensor, gold output.\n\n  Returns:\n    bleu: int, approx bleu score\n  \"\"\"\n  outputs = tf.to_int32(tf.argmax(predictions, axis=-1))\n  # Convert the outputs and labels to a [batch_size, input_length] tensor.\n  outputs = tf.squeeze(outputs, axis=[-1, -2])\n  labels = tf.squeeze(labels, axis=[-1, -2])\n\n  bleu = tf.py_func(compute_bleu, (labels, outputs), tf.float32)\n  return bleu, tf.constant(1.0)\n\n\nclass UnicodeRegex(object):\n  \"\"\"Ad-hoc hack to recognize all punctuation and symbols.\"\"\"\n\n  def __init__(self):\n    def _property_chars(prefix):\n      return ''.join(six.unichr(x) for x in range(sys.maxunicode)\n                     if unicodedata.category(six.unichr(x)).startswith(prefix))\n    punctuation = _property_chars('P')\n    self.nondigit_punct_re = re.compile(r'([^\\d])([' + punctuation + r'])')\n    self.punct_nondigit_re = re.compile(r'([' + punctuation + r'])([^\\d])')\n    self.symbol_re = re.compile('([' + _property_chars('S') + '])')\n\n\ndef bleu_tokenize(string):\n  r\"\"\"Tokenize a string following the official BLEU implementation.\n\n  See https://github.com/moses-smt/mosesdecoder/\"\n           \"blob/master/scripts/generic/mteval-v14.pl#L954-L983\n  In our case, the input string is expected to be just one line\n  and no HTML entities de-escaping is needed.\n  So we just tokenize on punctuation and symbols,\n  except when a punctuation is preceded and followed by a digit\n  (e.g. a comma/dot as a thousand/decimal separator).\n\n  Note that a numer (e.g. a year) followed by a dot at the end of sentence\n  is NOT tokenized,\n  i.e. the dot stays with the number because `s/(\\p{P})(\\P{N})/ $1 $2/g`\n  does not match this case (unless we add a space after each sentence).\n  However, this error is already in the original mteval-v14.pl\n  and we want to be consistent with it.\n\n  Args:\n    string: the input string\n\n  Returns:\n    a list of tokens\n  \"\"\"\n  uregex = UnicodeRegex()\n  string = uregex.nondigit_punct_re.sub(r'\\1 \\2 ', string)\n  string = uregex.punct_nondigit_re.sub(r' \\1 \\2', string)\n  string = uregex.symbol_re.sub(r' \\1 ', string)\n  return string.split()\n\n\ndef bleu_wrapper(ref_filename, hyp_filename, case_sensitive=False):\n  \"\"\"Compute BLEU for two files (reference and hypothesis translation).\"\"\"\n  ref_lines = open(ref_filename).read().splitlines()\n  hyp_lines = open(hyp_filename).read().splitlines()\n  assert len(ref_lines) == len(hyp_lines)\n  if not case_sensitive:\n    ref_lines = [x.lower() for x in ref_lines]\n    hyp_lines = [x.lower() for x in hyp_lines]\n  ref_tokens = [bleu_tokenize(x) for x in ref_lines]\n  hyp_tokens = [bleu_tokenize(x) for x in hyp_lines]\n  return compute_bleu(ref_tokens, hyp_tokens)\n\n\nStepFile = namedtuple('StepFile', 'filename mtime ctime steps')\n\n\ndef _read_stepfiles_list(path_prefix, path_suffix='.index', min_steps=0):\n  stepfiles = []\n  for filename in tf.gfile.Glob(path_prefix + '*-[0-9]*' + path_suffix):\n    basename = filename[:-len(path_suffix)] if len(path_suffix) else filename\n    try:\n      steps = int(basename.rsplit('-')[-1])\n    except ValueError:  # The -[0-9]* part is not an integer.\n      continue\n    if steps < min_steps:\n      continue\n    if not os.path.exists(filename):\n      tf.logging.info(filename + \" was deleted, so skipping it\")\n      continue\n    stepfiles.append(StepFile(basename, os.path.getmtime(filename),\n                              os.path.getctime(filename), steps))\n  return sorted(stepfiles, key=lambda x: -x.steps)\n\n\ndef stepfiles_iterator(path_prefix, wait_minutes=0, min_steps=0,\n                       path_suffix='.index', sleep_sec=10):\n  \"\"\"Continuously yield new files with steps in filename as they appear.\n\n  This is useful for checkpoint files or other files whose names differ just in an interger\n  marking the number of steps and match the wildcard path_prefix + '*-[0-9]*' + path_suffix.\n  Unlike `tf.contrib.training.checkpoints_iterator`, this\n  implementation always starts from the oldest files\n  (and it cannot miss any file). Note that the oldest checkpoint\n  may be deleted anytime by Tensorflow (if set up so). It is up to the user\n  to check that the files returned by this generator actually exist.\n  Args:\n    path_prefix: The directory + possible common filename prefix to the files.\n    path_suffix: Common filename suffix (after steps), including possible extension dot.\n    wait_minutes: The maximum amount of minutes to wait between files.\n    min_steps: Skip files with lower global step.\n    sleep_sec: How often to check for new files.\n  Yields:\n    named tuples (filename, mtime, ctime, steps) of the files as they arrive.\n  \"\"\"\n  # Wildcard D*-[0-9]* does not match D/x-1, so if D is a directory let path_prefix='D/'.\n  if not path_prefix.endswith(os.sep) and os.path.isdir(path_prefix):\n    path_prefix += os.sep\n  stepfiles = _read_stepfiles_list(path_prefix, path_suffix, min_steps)\n  tf.logging.info(\"Found %d files with steps: %s\"\n                  % (len(stepfiles), \", \".join(str(x.steps) for x in reversed(stepfiles))))\n  exit_time = time.time() + wait_minutes * 60\n  while True:\n    if not stepfiles and wait_minutes:\n      tf.logging.info('Waiting till %s if a new file matching %s*-[0-9]*%s appears'\n                      % (time.asctime(time.localtime(exit_time)), path_prefix, path_suffix))\n      while True:\n        stepfiles = _read_stepfiles_list(path_prefix, path_suffix, min_steps)\n        if stepfiles or time.time() > exit_time:\n          break\n        time.sleep(sleep_sec)\n    if not stepfiles:\n      return\n\n    stepfile = stepfiles.pop()\n    exit_time, min_steps = stepfile.ctime + wait_minutes * 60, stepfile.steps + 1\n    yield stepfile\n", "path": "tensor2tensor/utils/bleu_hook.py"}]}