We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Poor translation results with the Transformer
The Transformer model produces very bad translation results. Its implementation should be revised and fixed.

See also the reference implementation at https://github.com/tensorflow/tensor2tensor.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `opennmt/utils/transformer.py`
Content:
```
1 """Define functions related to the Google's Transformer model."""
2 
3 import tensorflow as tf
4 
5 
6 def scaled_dot_attention(queries,
7                          keys,
8                          values,
9                          mode,
10                          values_length=None,
11                          mask_future=False,
12                          dropout=0.0):
13   """Computes the scaled dot-product attention as described
14   in https://arxiv.org/abs/1706.03762.
15 
16   Args:
17     queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.
18     keys: The sequence use to calculate attention scores. A tensor of shape
19       :math:`[B, T_2, ...]`.
20     values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.
21     mode: A ``tf.estimator.ModeKeys`` mode.
22     values_length: The length of the values to attend.
23     mask_future: Mask attention to future positions.
24     dropout: The probability to drop units from the inputs.
25 
26   Returns:
27     A tuple ``(context vector, attention vector)``.
28   """
29   # Scaled dot-product between queries and keys.
30   dot = tf.matmul(queries, keys, transpose_b=True)
31   dot = tf.div(dot, tf.sqrt(tf.cast(tf.shape(keys)[-1], tf.float32)))
32 
33   if values_length is not None:
34     # Give no weight to illegal connections.
35     if mask_future:
36       # When masking the future, a position can only attend to previous timesteps.
37       mask = tf.map_fn(
38           lambda x: tf.sequence_mask(
39               tf.minimum(tf.range(tf.shape(values)[1]) + 1, x),
40               maxlen=tf.shape(values)[1],
41               dtype=tf.float32),
42           values_length,
43           dtype=tf.float32)
44     else:
45       # Otherwise, simply prevent attention on out-of-range positions.
46       mask = tf.sequence_mask(
47           values_length,
48           maxlen=tf.shape(values)[1],
49           dtype=tf.float32)
50       mask = tf.expand_dims(mask, axis=1)
51 
52     dot = dot * mask + ((1.0 - mask) * tf.float32.min)
53 
54   # Compute attention weights.
55   attn = tf.nn.softmax(dot)
56   attn = tf.layers.dropout(
57       attn,
58       rate=dropout,
59       training=mode == tf.estimator.ModeKeys.TRAIN)
60 
61   # Compute attention context.
62   context = tf.matmul(attn, values)
63 
64   return context, attn
65 
66 
67 def multi_head_attention(num_heads,
68                          queries,
69                          keys,
70                          values,
71                          mode,
72                          values_length=None,
73                          mask_future=False,
74                          dropout=0.0):
75   """Computes the multi-head attention as described in
76   https://arxiv.org/abs/1706.03762.
77 
78   Args:
79     num_heads: The number of attention heads.
80     queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.
81     keys: The sequence use to calculate attention scores. A tensor of shape
82       :math:`[B, T_2, ...]`.
83     values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.
84     mode: A ``tf.estimator.ModeKeys`` mode.
85     values_length: The length of the values to attend.
86     mask_future: Mask attention to future positions.
87     dropout: The probability to drop units from the inputs.
88 
89   Returns:
90     The concatenated attention context of each head.
91   """
92   input_dim = keys.get_shape().as_list()[-1]
93 
94   if input_dim % num_heads != 0:
95     raise ValueError("Multi head attention requires the input dimension to be a"
96                      " multiple of {}".format(num_heads))
97 
98   head_dim = input_dim / num_heads
99   heads = []
100 
101   for i in range(num_heads):
102     with tf.variable_scope("head_{}".format(i)):
103       # Project queries, keys and values to different and smaller subspaces.
104       queries_proj = tf.layers.conv1d(queries, head_dim, 1)
105       keys_proj = tf.layers.conv1d(keys, head_dim, 1)
106       values_proj = tf.layers.conv1d(values, head_dim, 1)
107 
108       head_i, _ = scaled_dot_attention(
109           queries_proj,
110           keys_proj,
111           values_proj,
112           mode,
113           values_length=values_length,
114           mask_future=mask_future,
115           dropout=dropout)
116 
117       heads.append(head_i)
118 
119   # Concatenate all heads output.
120   combined = tf.concat(heads, axis=2)
121   outputs = tf.layers.conv1d(combined, input_dim, 1)
122 
123   return outputs
124 
125 def feed_forward(x, inner_dim):
126   """Implements the Transformer's "Feed Forward" layer.
127 
128   .. math::
129 
130       ffn(x) = max(0, x*W_1 + b_1)*W_2 + b_2
131 
132   Args:
133     x: The input.
134     inner_dim: The number of units of the inner linear transformation.
135 
136   Returns:
137     The transformed input.
138   """
139   input_dim = x.get_shape().as_list()[-1]
140 
141   inner = tf.layers.conv1d(x, inner_dim, 1, activation=tf.nn.relu)
142   outer = tf.layers.conv1d(inner, input_dim, 1)
143 
144   return outer
145 
146 def add_and_norm(inputs,
147                  outputs,
148                  mode,
149                  dropout=0.1):
150   """Implements the Transformer's "Add & Norm" layer.
151 
152   Args:
153     inputs: The input of the previous layer.
154     outputs: The output of the previous layer.
155     mode: A ``tf.estimator.ModeKeys`` mode.
156     dropout: The probability to drop units in :obj:`outputs`.
157 
158   Returns:
159     The residual and normalized output.
160   """
161   outputs = tf.layers.dropout(
162       outputs,
163       rate=dropout,
164       training=mode == tf.estimator.ModeKeys.TRAIN)
165   outputs += inputs
166   outputs = tf.contrib.layers.layer_norm(outputs)
167   return outputs
168
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/opennmt/utils/transformer.py b/opennmt/utils/transformer.py
--- a/opennmt/utils/transformer.py
+++ b/opennmt/utils/transformer.py
@@ -163,5 +163,5 @@
       rate=dropout,
       training=mode == tf.estimator.ModeKeys.TRAIN)
   outputs += inputs
-  outputs = tf.contrib.layers.layer_norm(outputs)
+  outputs = tf.contrib.layers.layer_norm(outputs, begin_norm_axis=-1)
   return outputs

{"golden_diff": "diff --git a/opennmt/utils/transformer.py b/opennmt/utils/transformer.py\n--- a/opennmt/utils/transformer.py\n+++ b/opennmt/utils/transformer.py\n@@ -163,5 +163,5 @@\n       rate=dropout,\n       training=mode == tf.estimator.ModeKeys.TRAIN)\n   outputs += inputs\n-  outputs = tf.contrib.layers.layer_norm(outputs)\n+  outputs = tf.contrib.layers.layer_norm(outputs, begin_norm_axis=-1)\n   return outputs\n", "issue": "Poor translation results with the Transformer\nThe Transformer model produces very bad translation results. Its implementation should be revised and fixed.\r\n\r\nSee also the reference implementation at https://github.com/tensorflow/tensor2tensor.\n", "before_files": [{"content": "\"\"\"Define functions related to the Google's Transformer model.\"\"\"\n\nimport tensorflow as tf\n\n\ndef scaled_dot_attention(queries,\n                         keys,\n                         values,\n                         mode,\n                         values_length=None,\n                         mask_future=False,\n                         dropout=0.0):\n  \"\"\"Computes the scaled dot-product attention as described\n  in https://arxiv.org/abs/1706.03762.\n\n  Args:\n    queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.\n    keys: The sequence use to calculate attention scores. A tensor of shape\n      :math:`[B, T_2, ...]`.\n    values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    values_length: The length of the values to attend.\n    mask_future: Mask attention to future positions.\n    dropout: The probability to drop units from the inputs.\n\n  Returns:\n    A tuple ``(context vector, attention vector)``.\n  \"\"\"\n  # Scaled dot-product between queries and keys.\n  dot = tf.matmul(queries, keys, transpose_b=True)\n  dot = tf.div(dot, tf.sqrt(tf.cast(tf.shape(keys)[-1], tf.float32)))\n\n  if values_length is not None:\n    # Give no weight to illegal connections.\n    if mask_future:\n      # When masking the future, a position can only attend to previous timesteps.\n      mask = tf.map_fn(\n          lambda x: tf.sequence_mask(\n              tf.minimum(tf.range(tf.shape(values)[1]) + 1, x),\n              maxlen=tf.shape(values)[1],\n              dtype=tf.float32),\n          values_length,\n          dtype=tf.float32)\n    else:\n      # Otherwise, simply prevent attention on out-of-range positions.\n      mask = tf.sequence_mask(\n          values_length,\n          maxlen=tf.shape(values)[1],\n          dtype=tf.float32)\n      mask = tf.expand_dims(mask, axis=1)\n\n    dot = dot * mask + ((1.0 - mask) * tf.float32.min)\n\n  # Compute attention weights.\n  attn = tf.nn.softmax(dot)\n  attn = tf.layers.dropout(\n      attn,\n      rate=dropout,\n      training=mode == tf.estimator.ModeKeys.TRAIN)\n\n  # Compute attention context.\n  context = tf.matmul(attn, values)\n\n  return context, attn\n\n\ndef multi_head_attention(num_heads,\n                         queries,\n                         keys,\n                         values,\n                         mode,\n                         values_length=None,\n                         mask_future=False,\n                         dropout=0.0):\n  \"\"\"Computes the multi-head attention as described in\n  https://arxiv.org/abs/1706.03762.\n\n  Args:\n    num_heads: The number of attention heads.\n    queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.\n    keys: The sequence use to calculate attention scores. A tensor of shape\n      :math:`[B, T_2, ...]`.\n    values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    values_length: The length of the values to attend.\n    mask_future: Mask attention to future positions.\n    dropout: The probability to drop units from the inputs.\n\n  Returns:\n    The concatenated attention context of each head.\n  \"\"\"\n  input_dim = keys.get_shape().as_list()[-1]\n\n  if input_dim % num_heads != 0:\n    raise ValueError(\"Multi head attention requires the input dimension to be a\"\n                     \" multiple of {}\".format(num_heads))\n\n  head_dim = input_dim / num_heads\n  heads = []\n\n  for i in range(num_heads):\n    with tf.variable_scope(\"head_{}\".format(i)):\n      # Project queries, keys and values to different and smaller subspaces.\n      queries_proj = tf.layers.conv1d(queries, head_dim, 1)\n      keys_proj = tf.layers.conv1d(keys, head_dim, 1)\n      values_proj = tf.layers.conv1d(values, head_dim, 1)\n\n      head_i, _ = scaled_dot_attention(\n          queries_proj,\n          keys_proj,\n          values_proj,\n          mode,\n          values_length=values_length,\n          mask_future=mask_future,\n          dropout=dropout)\n\n      heads.append(head_i)\n\n  # Concatenate all heads output.\n  combined = tf.concat(heads, axis=2)\n  outputs = tf.layers.conv1d(combined, input_dim, 1)\n\n  return outputs\n\ndef feed_forward(x, inner_dim):\n  \"\"\"Implements the Transformer's \"Feed Forward\" layer.\n\n  .. math::\n\n      ffn(x) = max(0, x*W_1 + b_1)*W_2 + b_2\n\n  Args:\n    x: The input.\n    inner_dim: The number of units of the inner linear transformation.\n\n  Returns:\n    The transformed input.\n  \"\"\"\n  input_dim = x.get_shape().as_list()[-1]\n\n  inner = tf.layers.conv1d(x, inner_dim, 1, activation=tf.nn.relu)\n  outer = tf.layers.conv1d(inner, input_dim, 1)\n\n  return outer\n\ndef add_and_norm(inputs,\n                 outputs,\n                 mode,\n                 dropout=0.1):\n  \"\"\"Implements the Transformer's \"Add & Norm\" layer.\n\n  Args:\n    inputs: The input of the previous layer.\n    outputs: The output of the previous layer.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    dropout: The probability to drop units in :obj:`outputs`.\n\n  Returns:\n    The residual and normalized output.\n  \"\"\"\n  outputs = tf.layers.dropout(\n      outputs,\n      rate=dropout,\n      training=mode == tf.estimator.ModeKeys.TRAIN)\n  outputs += inputs\n  outputs = tf.contrib.layers.layer_norm(outputs)\n  return outputs\n", "path": "opennmt/utils/transformer.py"}], "after_files": [{"content": "\"\"\"Define functions related to the Google's Transformer model.\"\"\"\n\nimport tensorflow as tf\n\n\ndef scaled_dot_attention(queries,\n                         keys,\n                         values,\n                         mode,\n                         values_length=None,\n                         mask_future=False,\n                         dropout=0.0):\n  \"\"\"Computes the scaled dot-product attention as described\n  in https://arxiv.org/abs/1706.03762.\n\n  Args:\n    queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.\n    keys: The sequence use to calculate attention scores. A tensor of shape\n      :math:`[B, T_2, ...]`.\n    values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    values_length: The length of the values to attend.\n    mask_future: Mask attention to future positions.\n    dropout: The probability to drop units from the inputs.\n\n  Returns:\n    A tuple ``(context vector, attention vector)``.\n  \"\"\"\n  # Scaled dot-product between queries and keys.\n  dot = tf.matmul(queries, keys, transpose_b=True)\n  dot = tf.div(dot, tf.sqrt(tf.cast(tf.shape(keys)[-1], tf.float32)))\n\n  if values_length is not None:\n    # Give no weight to illegal connections.\n    if mask_future:\n      # When masking the future, a position can only attend to previous timesteps.\n      mask = tf.map_fn(\n          lambda x: tf.sequence_mask(\n              tf.minimum(tf.range(tf.shape(values)[1]) + 1, x),\n              maxlen=tf.shape(values)[1],\n              dtype=tf.float32),\n          values_length,\n          dtype=tf.float32)\n    else:\n      # Otherwise, simply prevent attention on out-of-range positions.\n      mask = tf.sequence_mask(\n          values_length,\n          maxlen=tf.shape(values)[1],\n          dtype=tf.float32)\n      mask = tf.expand_dims(mask, axis=1)\n\n    dot = dot * mask + ((1.0 - mask) * tf.float32.min)\n\n  # Compute attention weights.\n  attn = tf.nn.softmax(dot)\n  attn = tf.layers.dropout(\n      attn,\n      rate=dropout,\n      training=mode == tf.estimator.ModeKeys.TRAIN)\n\n  # Compute attention context.\n  context = tf.matmul(attn, values)\n\n  return context, attn\n\n\ndef multi_head_attention(num_heads,\n                         queries,\n                         keys,\n                         values,\n                         mode,\n                         values_length=None,\n                         mask_future=False,\n                         dropout=0.0):\n  \"\"\"Computes the multi-head attention as described in\n  https://arxiv.org/abs/1706.03762.\n\n  Args:\n    num_heads: The number of attention heads.\n    queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.\n    keys: The sequence use to calculate attention scores. A tensor of shape\n      :math:`[B, T_2, ...]`.\n    values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    values_length: The length of the values to attend.\n    mask_future: Mask attention to future positions.\n    dropout: The probability to drop units from the inputs.\n\n  Returns:\n    The concatenated attention context of each head.\n  \"\"\"\n  input_dim = keys.get_shape().as_list()[-1]\n\n  if input_dim % num_heads != 0:\n    raise ValueError(\"Multi head attention requires the input dimension to be a\"\n                     \" multiple of {}\".format(num_heads))\n\n  head_dim = input_dim / num_heads\n  heads = []\n\n  for i in range(num_heads):\n    with tf.variable_scope(\"head_{}\".format(i)):\n      # Project queries, keys and values to different and smaller subspaces.\n      queries_proj = tf.layers.conv1d(queries, head_dim, 1)\n      keys_proj = tf.layers.conv1d(keys, head_dim, 1)\n      values_proj = tf.layers.conv1d(values, head_dim, 1)\n\n      head_i, _ = scaled_dot_attention(\n          queries_proj,\n          keys_proj,\n          values_proj,\n          mode,\n          values_length=values_length,\n          mask_future=mask_future,\n          dropout=dropout)\n\n      heads.append(head_i)\n\n  # Concatenate all heads output.\n  combined = tf.concat(heads, axis=2)\n  outputs = tf.layers.conv1d(combined, input_dim, 1)\n\n  return outputs\n\ndef feed_forward(x, inner_dim):\n  \"\"\"Implements the Transformer's \"Feed Forward\" layer.\n\n  .. math::\n\n      ffn(x) = max(0, x*W_1 + b_1)*W_2 + b_2\n\n  Args:\n    x: The input.\n    inner_dim: The number of units of the inner linear transformation.\n\n  Returns:\n    The transformed input.\n  \"\"\"\n  input_dim = x.get_shape().as_list()[-1]\n\n  inner = tf.layers.conv1d(x, inner_dim, 1, activation=tf.nn.relu)\n  outer = tf.layers.conv1d(inner, input_dim, 1)\n\n  return outer\n\ndef add_and_norm(inputs,\n                 outputs,\n                 mode,\n                 dropout=0.1):\n  \"\"\"Implements the Transformer's \"Add & Norm\" layer.\n\n  Args:\n    inputs: The input of the previous layer.\n    outputs: The output of the previous layer.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    dropout: The probability to drop units in :obj:`outputs`.\n\n  Returns:\n    The residual and normalized output.\n  \"\"\"\n  outputs = tf.layers.dropout(\n      outputs,\n      rate=dropout,\n      training=mode == tf.estimator.ModeKeys.TRAIN)\n  outputs += inputs\n  outputs = tf.contrib.layers.layer_norm(outputs, begin_norm_axis=-1)\n  return outputs\n", "path": "opennmt/utils/transformer.py"}]}