We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Naming inconsistency between Gpt2 and XLMRoberta
We handle case differently in our current naming for `Gpt2` and `XLMRoberta`. We should align on either:
 - `XLMRoberta` and `GPT2`
 - `XlmRoberta` and `Gpt2`

Once we decided on the naming we want, this can just be a simple rename. We should probably do this before the next major release.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `keras_nlp/models/__init__.py`
Content:
```
1 # Copyright 2022 The KerasNLP Authors
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     https://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 from keras_nlp.models.bert.bert_models import Bert
16 from keras_nlp.models.bert.bert_preprocessing import BertPreprocessor
17 from keras_nlp.models.bert.bert_tasks import BertClassifier
18 from keras_nlp.models.distilbert.distilbert_models import DistilBert
19 from keras_nlp.models.distilbert.distilbert_preprocessing import (
20     DistilBertPreprocessor,
21 )
22 from keras_nlp.models.gpt2.gpt2_models import Gpt2
23 from keras_nlp.models.roberta.roberta_models import Roberta
24 from keras_nlp.models.roberta.roberta_tasks import RobertaClassifier
25 from keras_nlp.models.xlm_roberta.xlm_roberta_models import XLMRoberta
26 from keras_nlp.models.xlm_roberta.xlm_roberta_preprocessing import (
27     XLMRobertaPreprocessor,
28 )
29
```

Path: `keras_nlp/models/gpt2/gpt2_models.py`
Content:
```
1 # Copyright 2022 The KerasNLP Authors
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     https://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 """GPT-2 backbone models."""
16 
17 import tensorflow as tf
18 from tensorflow import keras
19 
20 from keras_nlp.layers import PositionEmbedding
21 from keras_nlp.layers import TransformerDecoder
22 
23 
24 def _gpt_2_kernel_initializer(stddev=0.02):
25     return keras.initializers.RandomNormal(stddev=stddev)
26 
27 
28 @keras.utils.register_keras_serializable(package="keras_nlp")
29 class Gpt2(keras.Model):
30     """GPT-2 core network with hyperparameters.
31 
32     This network implements a Transformer-based decoder network,
33     Generative Pretrained Transformer-2 (GPT-2), as described in
34     ["Language Models are Unsupervised Multitask Learners"](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf).
35     It includes the embedding lookups and transformer layers.
36 
37     The default constructor gives a fully customizable, randomly initalized
38     GPT-2 model with any number of layers, heads, and embedding
39     dimensions. To load preset architectures and weights, use the `from_presets`
40     constructor.
41 
42     Args:
43         vocabulary_size: int. The size of the token vocabulary.
44         num_layers: int. The number of transformer layers.
45         num_heads: int. The number of attention heads for each transformer.
46             The hidden size must be divisible by the number of attention heads.
47         hidden_dim: int. The size of the transformer encoding and pooler layers.
48         intermediate_dim: int. The output dimension of the first Dense layer in
49             a two-layer feedforward network for each transformer.
50         dropout: float. Dropout probability for the Transformer encoder.
51         max_sequence_length: int. The maximum sequence length that this encoder
52             can consume. If None, `max_sequence_length` uses the value from
53             sequence length. This determines the variable shape for positional
54             embeddings.
55 
56     Example usage:
57     ```python
58     input_data = {
59         "token_ids": tf.random.uniform(
60             shape=(1, 12), dtype=tf.int64, maxval=model.vocabulary_size
61         ),
62         "padding_mask": tf.constant(
63             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0], shape=(1, 12)
64         ),
65     }
66 
67     # Randomly initialized GPT-2 decoder
68     model = keras_nlp.models.Gpt2(
69         vocabulary_size=50257,
70         num_layers=12,
71         num_heads=12,
72         hidden_dim=768,
73         intermediate_dim=3072,
74         max_sequence_length=1024,
75     )
76 
77     # Call the model on the input data.
78     output = model(input_data)
79     ```
80     """
81 
82     def __init__(
83         self,
84         vocabulary_size,
85         num_layers,
86         num_heads,
87         hidden_dim,
88         intermediate_dim,
89         dropout=0.1,
90         max_sequence_length=1024,
91         **kwargs,
92     ):
93 
94         # Inputs
95         token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids")
96         padding_mask = keras.Input(
97             shape=(None,), dtype="int32", name="padding_mask"
98         )
99 
100         # Embed tokens, positions.
101         token_embedding = keras.layers.Embedding(
102             input_dim=vocabulary_size,
103             output_dim=hidden_dim,
104             embeddings_initializer=_gpt_2_kernel_initializer(stddev=0.01),
105             name="token_embedding",
106         )(token_ids)
107 
108         # Can't use `TokenAndPositionEmbedding` layer here because of different
109         # initializers.
110         position_embedding = PositionEmbedding(
111             initializer=_gpt_2_kernel_initializer(stddev=0.02),
112             sequence_length=max_sequence_length,
113             name="position_embedding",
114         )(token_embedding)
115 
116         # Sum and apply dropout to embeddings.
117         x = keras.layers.Add()((token_embedding, position_embedding))
118         x = keras.layers.Dropout(
119             dropout,
120             name="embeddings_dropout",
121         )(x)
122 
123         # Apply successive transformer decoder blocks.
124         for i in range(num_layers):
125             x = TransformerDecoder(
126                 intermediate_dim=intermediate_dim,
127                 num_heads=num_heads,
128                 dropout=dropout,
129                 activation=lambda x: keras.activations.gelu(
130                     x, approximate=True
131                 ),
132                 layer_norm_epsilon=1e-05,
133                 kernel_initializer=_gpt_2_kernel_initializer(stddev=0.02),
134                 normalize_first=True,
135                 name=f"transformer_layer_{i}",
136             )(x, decoder_padding_mask=padding_mask)
137 
138         sequence_output = keras.layers.LayerNormalization(
139             name="layer_norm",
140             axis=-1,
141             epsilon=1e-05,
142             dtype=tf.float32,
143         )(x)
144 
145         # Set default for `name` if none given
146         if "name" not in kwargs:
147             kwargs["name"] = "backbone"
148 
149         # Instantiate using Functional API Model constructor
150         super().__init__(
151             inputs={
152                 "token_ids": token_ids,
153                 "padding_mask": padding_mask,
154             },
155             outputs=sequence_output,
156             **kwargs,
157         )
158         # All references to `self` below this line
159         self.vocabulary_size = vocabulary_size
160         self.num_layers = num_layers
161         self.num_heads = num_heads
162         self.hidden_dim = hidden_dim
163         self.intermediate_dim = intermediate_dim
164         self.dropout = dropout
165         self.max_sequence_length = max_sequence_length
166 
167     def get_config(self):
168         return {
169             "vocabulary_size": self.vocabulary_size,
170             "num_layers": self.num_layers,
171             "num_heads": self.num_heads,
172             "hidden_dim": self.hidden_dim,
173             "intermediate_dim": self.intermediate_dim,
174             "dropout": self.dropout,
175             "max_sequence_length": self.max_sequence_length,
176             "name": self.name,
177             "trainable": self.trainable,
178         }
179 
180     @classmethod
181     def from_config(cls, config):
182         return cls(**config)
183 
184     @classmethod
185     def from_preset(
186         cls,
187         preset,
188         load_weights=True,
189         **kwargs,
190     ):
191         raise NotImplementedError
192
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py
--- a/keras_nlp/models/__init__.py
+++ b/keras_nlp/models/__init__.py
@@ -19,7 +19,7 @@
 from keras_nlp.models.distilbert.distilbert_preprocessing import (
     DistilBertPreprocessor,
 )
-from keras_nlp.models.gpt2.gpt2_models import Gpt2
+from keras_nlp.models.gpt2.gpt2_models import GPT2
 from keras_nlp.models.roberta.roberta_models import Roberta
 from keras_nlp.models.roberta.roberta_tasks import RobertaClassifier
 from keras_nlp.models.xlm_roberta.xlm_roberta_models import XLMRoberta
diff --git a/keras_nlp/models/gpt2/gpt2_models.py b/keras_nlp/models/gpt2/gpt2_models.py
--- a/keras_nlp/models/gpt2/gpt2_models.py
+++ b/keras_nlp/models/gpt2/gpt2_models.py
@@ -26,7 +26,7 @@
 
 
 @keras.utils.register_keras_serializable(package="keras_nlp")
-class Gpt2(keras.Model):
+class GPT2(keras.Model):
     """GPT-2 core network with hyperparameters.
 
     This network implements a Transformer-based decoder network,
@@ -65,7 +65,7 @@
     }
 
     # Randomly initialized GPT-2 decoder
-    model = keras_nlp.models.Gpt2(
+    model = keras_nlp.models.GPT2(
         vocabulary_size=50257,
         num_layers=12,
         num_heads=12,

{"golden_diff": "diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py\n--- a/keras_nlp/models/__init__.py\n+++ b/keras_nlp/models/__init__.py\n@@ -19,7 +19,7 @@\n from keras_nlp.models.distilbert.distilbert_preprocessing import (\n     DistilBertPreprocessor,\n )\n-from keras_nlp.models.gpt2.gpt2_models import Gpt2\n+from keras_nlp.models.gpt2.gpt2_models import GPT2\n from keras_nlp.models.roberta.roberta_models import Roberta\n from keras_nlp.models.roberta.roberta_tasks import RobertaClassifier\n from keras_nlp.models.xlm_roberta.xlm_roberta_models import XLMRoberta\ndiff --git a/keras_nlp/models/gpt2/gpt2_models.py b/keras_nlp/models/gpt2/gpt2_models.py\n--- a/keras_nlp/models/gpt2/gpt2_models.py\n+++ b/keras_nlp/models/gpt2/gpt2_models.py\n@@ -26,7 +26,7 @@\n \n \n @keras.utils.register_keras_serializable(package=\"keras_nlp\")\n-class Gpt2(keras.Model):\n+class GPT2(keras.Model):\n     \"\"\"GPT-2 core network with hyperparameters.\n \n     This network implements a Transformer-based decoder network,\n@@ -65,7 +65,7 @@\n     }\n \n     # Randomly initialized GPT-2 decoder\n-    model = keras_nlp.models.Gpt2(\n+    model = keras_nlp.models.GPT2(\n         vocabulary_size=50257,\n         num_layers=12,\n         num_heads=12,\n", "issue": "Naming inconsistency between Gpt2 and XLMRoberta\nWe handle case differently in our current naming for `Gpt2` and `XLMRoberta`. We should align on either:\r\n - `XLMRoberta` and `GPT2`\r\n - `XlmRoberta` and `Gpt2`\r\n\r\nOnce we decided on the naming we want, this can just be a simple rename. We should probably do this before the next major release.\n", "before_files": [{"content": "# Copyright 2022 The KerasNLP Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom keras_nlp.models.bert.bert_models import Bert\nfrom keras_nlp.models.bert.bert_preprocessing import BertPreprocessor\nfrom keras_nlp.models.bert.bert_tasks import BertClassifier\nfrom keras_nlp.models.distilbert.distilbert_models import DistilBert\nfrom keras_nlp.models.distilbert.distilbert_preprocessing import (\n    DistilBertPreprocessor,\n)\nfrom keras_nlp.models.gpt2.gpt2_models import Gpt2\nfrom keras_nlp.models.roberta.roberta_models import Roberta\nfrom keras_nlp.models.roberta.roberta_tasks import RobertaClassifier\nfrom keras_nlp.models.xlm_roberta.xlm_roberta_models import XLMRoberta\nfrom keras_nlp.models.xlm_roberta.xlm_roberta_preprocessing import (\n    XLMRobertaPreprocessor,\n)\n", "path": "keras_nlp/models/__init__.py"}, {"content": "# Copyright 2022 The KerasNLP Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"GPT-2 backbone models.\"\"\"\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom keras_nlp.layers import PositionEmbedding\nfrom keras_nlp.layers import TransformerDecoder\n\n\ndef _gpt_2_kernel_initializer(stddev=0.02):\n    return keras.initializers.RandomNormal(stddev=stddev)\n\n\n@keras.utils.register_keras_serializable(package=\"keras_nlp\")\nclass Gpt2(keras.Model):\n    \"\"\"GPT-2 core network with hyperparameters.\n\n    This network implements a Transformer-based decoder network,\n    Generative Pretrained Transformer-2 (GPT-2), as described in\n    [\"Language Models are Unsupervised Multitask Learners\"](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf).\n    It includes the embedding lookups and transformer layers.\n\n    The default constructor gives a fully customizable, randomly initalized\n    GPT-2 model with any number of layers, heads, and embedding\n    dimensions. To load preset architectures and weights, use the `from_presets`\n    constructor.\n\n    Args:\n        vocabulary_size: int. The size of the token vocabulary.\n        num_layers: int. The number of transformer layers.\n        num_heads: int. The number of attention heads for each transformer.\n            The hidden size must be divisible by the number of attention heads.\n        hidden_dim: int. The size of the transformer encoding and pooler layers.\n        intermediate_dim: int. The output dimension of the first Dense layer in\n            a two-layer feedforward network for each transformer.\n        dropout: float. Dropout probability for the Transformer encoder.\n        max_sequence_length: int. The maximum sequence length that this encoder\n            can consume. If None, `max_sequence_length` uses the value from\n            sequence length. This determines the variable shape for positional\n            embeddings.\n\n    Example usage:\n    ```python\n    input_data = {\n        \"token_ids\": tf.random.uniform(\n            shape=(1, 12), dtype=tf.int64, maxval=model.vocabulary_size\n        ),\n        \"padding_mask\": tf.constant(\n            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0], shape=(1, 12)\n        ),\n    }\n\n    # Randomly initialized GPT-2 decoder\n    model = keras_nlp.models.Gpt2(\n        vocabulary_size=50257,\n        num_layers=12,\n        num_heads=12,\n        hidden_dim=768,\n        intermediate_dim=3072,\n        max_sequence_length=1024,\n    )\n\n    # Call the model on the input data.\n    output = model(input_data)\n    ```\n    \"\"\"\n\n    def __init__(\n        self,\n        vocabulary_size,\n        num_layers,\n        num_heads,\n        hidden_dim,\n        intermediate_dim,\n        dropout=0.1,\n        max_sequence_length=1024,\n        **kwargs,\n    ):\n\n        # Inputs\n        token_ids = keras.Input(shape=(None,), dtype=\"int32\", name=\"token_ids\")\n        padding_mask = keras.Input(\n            shape=(None,), dtype=\"int32\", name=\"padding_mask\"\n        )\n\n        # Embed tokens, positions.\n        token_embedding = keras.layers.Embedding(\n            input_dim=vocabulary_size,\n            output_dim=hidden_dim,\n            embeddings_initializer=_gpt_2_kernel_initializer(stddev=0.01),\n            name=\"token_embedding\",\n        )(token_ids)\n\n        # Can't use `TokenAndPositionEmbedding` layer here because of different\n        # initializers.\n        position_embedding = PositionEmbedding(\n            initializer=_gpt_2_kernel_initializer(stddev=0.02),\n            sequence_length=max_sequence_length,\n            name=\"position_embedding\",\n        )(token_embedding)\n\n        # Sum and apply dropout to embeddings.\n        x = keras.layers.Add()((token_embedding, position_embedding))\n        x = keras.layers.Dropout(\n            dropout,\n            name=\"embeddings_dropout\",\n        )(x)\n\n        # Apply successive transformer decoder blocks.\n        for i in range(num_layers):\n            x = TransformerDecoder(\n                intermediate_dim=intermediate_dim,\n                num_heads=num_heads,\n                dropout=dropout,\n                activation=lambda x: keras.activations.gelu(\n                    x, approximate=True\n                ),\n                layer_norm_epsilon=1e-05,\n                kernel_initializer=_gpt_2_kernel_initializer(stddev=0.02),\n                normalize_first=True,\n                name=f\"transformer_layer_{i}\",\n            )(x, decoder_padding_mask=padding_mask)\n\n        sequence_output = keras.layers.LayerNormalization(\n            name=\"layer_norm\",\n            axis=-1,\n            epsilon=1e-05,\n            dtype=tf.float32,\n        )(x)\n\n        # Set default for `name` if none given\n        if \"name\" not in kwargs:\n            kwargs[\"name\"] = \"backbone\"\n\n        # Instantiate using Functional API Model constructor\n        super().__init__(\n            inputs={\n                \"token_ids\": token_ids,\n                \"padding_mask\": padding_mask,\n            },\n            outputs=sequence_output,\n            **kwargs,\n        )\n        # All references to `self` below this line\n        self.vocabulary_size = vocabulary_size\n        self.num_layers = num_layers\n        self.num_heads = num_heads\n        self.hidden_dim = hidden_dim\n        self.intermediate_dim = intermediate_dim\n        self.dropout = dropout\n        self.max_sequence_length = max_sequence_length\n\n    def get_config(self):\n        return {\n            \"vocabulary_size\": self.vocabulary_size,\n            \"num_layers\": self.num_layers,\n            \"num_heads\": self.num_heads,\n            \"hidden_dim\": self.hidden_dim,\n            \"intermediate_dim\": self.intermediate_dim,\n            \"dropout\": self.dropout,\n            \"max_sequence_length\": self.max_sequence_length,\n            \"name\": self.name,\n            \"trainable\": self.trainable,\n        }\n\n    @classmethod\n    def from_config(cls, config):\n        return cls(**config)\n\n    @classmethod\n    def from_preset(\n        cls,\n        preset,\n        load_weights=True,\n        **kwargs,\n    ):\n        raise NotImplementedError\n", "path": "keras_nlp/models/gpt2/gpt2_models.py"}], "after_files": [{"content": "# Copyright 2022 The KerasNLP Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom keras_nlp.models.bert.bert_models import Bert\nfrom keras_nlp.models.bert.bert_preprocessing import BertPreprocessor\nfrom keras_nlp.models.bert.bert_tasks import BertClassifier\nfrom keras_nlp.models.distilbert.distilbert_models import DistilBert\nfrom keras_nlp.models.distilbert.distilbert_preprocessing import (\n    DistilBertPreprocessor,\n)\nfrom keras_nlp.models.gpt2.gpt2_models import GPT2\nfrom keras_nlp.models.roberta.roberta_models import Roberta\nfrom keras_nlp.models.roberta.roberta_tasks import RobertaClassifier\nfrom keras_nlp.models.xlm_roberta.xlm_roberta_models import XLMRoberta\nfrom keras_nlp.models.xlm_roberta.xlm_roberta_preprocessing import (\n    XLMRobertaPreprocessor,\n)\n", "path": "keras_nlp/models/__init__.py"}, {"content": "# Copyright 2022 The KerasNLP Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"GPT-2 backbone models.\"\"\"\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom keras_nlp.layers import PositionEmbedding\nfrom keras_nlp.layers import TransformerDecoder\n\n\ndef _gpt_2_kernel_initializer(stddev=0.02):\n    return keras.initializers.RandomNormal(stddev=stddev)\n\n\n@keras.utils.register_keras_serializable(package=\"keras_nlp\")\nclass GPT2(keras.Model):\n    \"\"\"GPT-2 core network with hyperparameters.\n\n    This network implements a Transformer-based decoder network,\n    Generative Pretrained Transformer-2 (GPT-2), as described in\n    [\"Language Models are Unsupervised Multitask Learners\"](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf).\n    It includes the embedding lookups and transformer layers.\n\n    The default constructor gives a fully customizable, randomly initalized\n    GPT-2 model with any number of layers, heads, and embedding\n    dimensions. To load preset architectures and weights, use the `from_presets`\n    constructor.\n\n    Args:\n        vocabulary_size: int. The size of the token vocabulary.\n        num_layers: int. The number of transformer layers.\n        num_heads: int. The number of attention heads for each transformer.\n            The hidden size must be divisible by the number of attention heads.\n        hidden_dim: int. The size of the transformer encoding and pooler layers.\n        intermediate_dim: int. The output dimension of the first Dense layer in\n            a two-layer feedforward network for each transformer.\n        dropout: float. Dropout probability for the Transformer encoder.\n        max_sequence_length: int. The maximum sequence length that this encoder\n            can consume. If None, `max_sequence_length` uses the value from\n            sequence length. This determines the variable shape for positional\n            embeddings.\n\n    Example usage:\n    ```python\n    input_data = {\n        \"token_ids\": tf.random.uniform(\n            shape=(1, 12), dtype=tf.int64, maxval=model.vocabulary_size\n        ),\n        \"padding_mask\": tf.constant(\n            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0], shape=(1, 12)\n        ),\n    }\n\n    # Randomly initialized GPT-2 decoder\n    model = keras_nlp.models.GPT2(\n        vocabulary_size=50257,\n        num_layers=12,\n        num_heads=12,\n        hidden_dim=768,\n        intermediate_dim=3072,\n        max_sequence_length=1024,\n    )\n\n    # Call the model on the input data.\n    output = model(input_data)\n    ```\n    \"\"\"\n\n    def __init__(\n        self,\n        vocabulary_size,\n        num_layers,\n        num_heads,\n        hidden_dim,\n        intermediate_dim,\n        dropout=0.1,\n        max_sequence_length=1024,\n        **kwargs,\n    ):\n\n        # Inputs\n        token_ids = keras.Input(shape=(None,), dtype=\"int32\", name=\"token_ids\")\n        padding_mask = keras.Input(\n            shape=(None,), dtype=\"int32\", name=\"padding_mask\"\n        )\n\n        # Embed tokens, positions.\n        token_embedding = keras.layers.Embedding(\n            input_dim=vocabulary_size,\n            output_dim=hidden_dim,\n            embeddings_initializer=_gpt_2_kernel_initializer(stddev=0.01),\n            name=\"token_embedding\",\n        )(token_ids)\n\n        # Can't use `TokenAndPositionEmbedding` layer here because of different\n        # initializers.\n        position_embedding = PositionEmbedding(\n            initializer=_gpt_2_kernel_initializer(stddev=0.02),\n            sequence_length=max_sequence_length,\n            name=\"position_embedding\",\n        )(token_embedding)\n\n        # Sum and apply dropout to embeddings.\n        x = keras.layers.Add()((token_embedding, position_embedding))\n        x = keras.layers.Dropout(\n            dropout,\n            name=\"embeddings_dropout\",\n        )(x)\n\n        # Apply successive transformer decoder blocks.\n        for i in range(num_layers):\n            x = TransformerDecoder(\n                intermediate_dim=intermediate_dim,\n                num_heads=num_heads,\n                dropout=dropout,\n                activation=lambda x: keras.activations.gelu(\n                    x, approximate=True\n                ),\n                layer_norm_epsilon=1e-05,\n                kernel_initializer=_gpt_2_kernel_initializer(stddev=0.02),\n                normalize_first=True,\n                name=f\"transformer_layer_{i}\",\n            )(x, decoder_padding_mask=padding_mask)\n\n        sequence_output = keras.layers.LayerNormalization(\n            name=\"layer_norm\",\n            axis=-1,\n            epsilon=1e-05,\n            dtype=tf.float32,\n        )(x)\n\n        # Set default for `name` if none given\n        if \"name\" not in kwargs:\n            kwargs[\"name\"] = \"backbone\"\n\n        # Instantiate using Functional API Model constructor\n        super().__init__(\n            inputs={\n                \"token_ids\": token_ids,\n                \"padding_mask\": padding_mask,\n            },\n            outputs=sequence_output,\n            **kwargs,\n        )\n        # All references to `self` below this line\n        self.vocabulary_size = vocabulary_size\n        self.num_layers = num_layers\n        self.num_heads = num_heads\n        self.hidden_dim = hidden_dim\n        self.intermediate_dim = intermediate_dim\n        self.dropout = dropout\n        self.max_sequence_length = max_sequence_length\n\n    def get_config(self):\n        return {\n            \"vocabulary_size\": self.vocabulary_size,\n            \"num_layers\": self.num_layers,\n            \"num_heads\": self.num_heads,\n            \"hidden_dim\": self.hidden_dim,\n            \"intermediate_dim\": self.intermediate_dim,\n            \"dropout\": self.dropout,\n            \"max_sequence_length\": self.max_sequence_length,\n            \"name\": self.name,\n            \"trainable\": self.trainable,\n        }\n\n    @classmethod\n    def from_config(cls, config):\n        return cls(**config)\n\n    @classmethod\n    def from_preset(\n        cls,\n        preset,\n        load_weights=True,\n        **kwargs,\n    ):\n        raise NotImplementedError\n", "path": "keras_nlp/models/gpt2/gpt2_models.py"}]}