We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Fix rank 1 outputs for WordPieceTokenizer
Right now, a scalar string inputs to the tokenizer will turn into rank 2 outputs, which we do not want. Scalar inputs should result in rank 1 outputs.

See this colab for a reproduction of the issue:
https://colab.research.google.com/gist/mattdangerw/8e1fdbebadf0ff6ff2dc065b2e00db65/rank1wordpiece.ipynb

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `keras_nlp/tokenizers/word_piece_tokenizer.py`
Content:
```
1 # Copyright 2022 The KerasNLP Authors
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     https://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 from typing import Any
16 from typing import Dict
17 from typing import Iterable
18 from typing import List
19 from typing import Union
20 
21 import tensorflow as tf
22 import tensorflow_text as tf_text
23 
24 from keras_nlp.tokenizers import tokenizer
25 
26 # Matches whitespace and control characters.
27 WHITESPACE_REGEX = r"|".join(
28     [
29         r"\s",
30         # Invisible control characters
31         r"\p{Cc}",
32         r"\p{Cf}",
33     ]
34 )
35 
36 # Matches punctuation compatible with the original bert implementation.
37 PUNCTUATION_REGEX = r"|".join(
38     [
39         # Treat all non-letter/number ASCII as punctuation.
40         # Characters such as "^", "$", and "`" are not in the Unicode
41         # Punctuation class but we treat them as punctuation anyways.
42         r"[!-/]",
43         r"[:-@]",
44         r"[\[-`]",
45         r"[{-~]",
46         # Unicode punctuation class.
47         r"[\p{P}]",
48         # More unicode ranges.
49         r"[\x{4E00}-\x{9FFF}]",
50         r"[\x{3400}-\x{4DBF}]",
51         r"[\x{20000}-\x{2A6DF}]",
52         r"[\x{2A700}-\x{2B73F}]",
53         r"[\x{2B740}-\x{2B81F}]",
54         r"[\x{2B820}-\x{2CEAF}]",
55         r"[\x{F900}-\x{FAFF}]",
56         r"[\x{2F800}-\x{2FA1F}]",
57     ]
58 )
59 
60 # Matches both whitespace and punctuation.
61 WHITESPACE_AND_PUNCTUATION_REGEX = r"|".join(
62     [
63         WHITESPACE_REGEX,
64         PUNCTUATION_REGEX,
65     ]
66 )
67 
68 
69 class WordPieceTokenizer(tokenizer.Tokenizer):
70     """A word piece tokenizer layer.
71 
72     This layer provides an efficient, in graph, implementation of the WordPiece
73     algorithm used by BERT and other models.
74 
75     To make this layer more useful out of the box, the layer will pre-tokenize
76     the input, which will optionally lower-case, strip accents, and split the
77     input on whitespace and punctuation. Each of these pre-tokenization steps is
78     not reversible. The `detokenize` method will join words with a space, and
79     will not invert `tokenize` exactly.
80 
81     If a more custom pre-tokenization step is desired, the layer can be
82     configured to apply only the strict WordPiece algorithm by passing
83     `lowercase=False`, `strip_accents=False` and `split_pattern=None`. In
84     this case, inputs should be pre-split string tensors or ragged tensors.
85 
86     By default, the layer will output a `tf.RaggedTensor` where the last
87     dimension of the output is ragged after whitespace splitting and sub-word
88     tokenizing. If `sequence_length` is set, the layer will output a dense
89     `tf.Tensor` where all inputs have been padded or truncated to
90     `sequence_length`. The output dtype can be controlled via the `dtype`
91     argument, which should be either an integer or string type.
92 
93     Args:
94         vocabulary: A list of strings or a string string filename path. If
95             passing a list, each element of the list should be a single word
96             piece token string. If passing a filename, the file should be a
97             plain text file containing a single word piece token per line.
98         sequence_length: If set, the output will be converted to a dense
99             tensor and padded/trimmed so all outputs are of sequence_length.
100         lowercase: If true, the input text will be first lowered before
101             tokenization.
102         strip_accents: If true, all accent marks will be removed from text
103             before tokenization.
104         split_pattern: A regex pattern to match delimiters to split, or None
105             indicating that the input is pre-split and no splitting should be
106             performed. By default, all whitespace and punctuation marks will
107             be split on.
108         keep_pattern: A regex pattern of delimiters contained in the
109             `split_pattern` of delimeters that should be kept as independent
110             tokens. By default, all punctuation marks will be kept as tokens.
111         suffix_indicator: The characters prepended to a wordpiece to indicate
112             that it is a suffix to another subword.
113         oov_token: The string value to substitute for an unknown token. It
114             must be included in the vocab.
115 
116     References:
117         - [Schuster and Nakajima, 2012](https://research.google/pubs/pub37842/)
118         - [Song et al., 2020](https://arxiv.org/abs/2012.15524)
119 
120     Examples:
121 
122     Ragged outputs.
123     >>> vocab = ["[UNK]", "the", "qu", "##ick", "br", "##own", "fox", "."]
124     >>> inputs = ["The quick brown fox."]
125     >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab)
126     >>> tokenizer(inputs)
127     <tf.RaggedTensor [[1, 2, 3, 4, 5, 6, 7]]>
128 
129     Dense outputs.
130     >>> vocab = ["[UNK]", "the", "qu", "##ick", "br", "##own", "fox", "."]
131     >>> inputs = ["The quick brown fox."]
132     >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(
133     ...     vocabulary=vocab, sequence_length=10)
134     >>> tokenizer(inputs)
135     <tf.Tensor: shape=(1, 10), dtype=int32,
136         numpy=array([[1, 2, 3, 4, 5, 6, 7, 0, 0, 0]]>
137 
138     String output.
139     >>> vocab = ["[UNK]", "the", "qu", "##ick", "br", "##own", "fox", "."]
140     >>> inputs = ["The quick brown fox."]
141     >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(
142     ...     vocabulary=vocab, dtype="string")
143     >>> tokenizer(inputs)
144     <tf.RaggedTensor [[b'the', b'qu', b'##ick', b'br', b'##own', b'fox', b'.']]>
145 
146     Detokenization.
147     >>> vocab = ["[UNK]", "the", "qu", "##ick", "br", "##own", "fox", "."]
148     >>> inputs = ["The quick brown fox."]
149     >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab)
150     >>> tokenizer.detokenize(tokenizer.tokenize(inputs))
151     <tf.Tensor: shape=(1,), dtype=string,
152         numpy=array([b"the quick brown fox ."], dtype=object)>
153 
154     Custom splitting.
155     >>> vocab = ["[UNK]", "fox", ","]
156     >>> inputs = ["fox,,fox,fox"]
157     >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab,
158     ...     split_pattern=",", keep_pattern=",", dtype='string')(inputs)
159     <tf.RaggedTensor [[b'fox', b',', b',', b'fox', b',', b'fox']]>
160     >>> keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab,
161     ...     split_pattern=",", keep_pattern="", dtype='string')(inputs)
162     <tf.RaggedTensor [[b'fox', b'fox', b'fox']]>
163     """
164 
165     def __init__(
166         self,
167         vocabulary: Union[Iterable[str], str] = None,
168         sequence_length: int = None,
169         lowercase: bool = True,
170         strip_accents: bool = True,
171         split_pattern: str = WHITESPACE_AND_PUNCTUATION_REGEX,
172         keep_pattern: str = PUNCTUATION_REGEX,
173         suffix_indicator: str = "##",
174         oov_token: str = "[UNK]",
175         **kwargs,
176     ) -> None:
177         # Check dtype and provide a default.
178         if "dtype" not in kwargs or kwargs["dtype"] is None:
179             kwargs["dtype"] = tf.int32
180         else:
181             dtype = tf.dtypes.as_dtype(kwargs["dtype"])
182             if not dtype.is_integer and dtype != tf.string:
183                 raise ValueError(
184                     "Output dtype must be an integer type of a string. "
185                     f"Received: dtype={dtype}"
186                 )
187 
188         super().__init__(**kwargs)
189 
190         if isinstance(vocabulary, str):
191             self._vocab = [
192                 line.rstrip() for line in tf.io.gfile.GFile(vocabulary)
193             ]
194         elif isinstance(vocabulary, Iterable):
195             # Make a copy.
196             self._vocab = list(vocabulary)
197         else:
198             raise ValueError(
199                 "Vocabulary must be an file path or list of terms. "
200                 f"Received: vocabulary={vocabulary}"
201             )
202 
203         self._sequence_length = sequence_length
204         self._lowercase = lowercase
205         self._strip_accents = strip_accents
206         self._split_pattern = split_pattern
207         self._keep_pattern = keep_pattern
208         self._suffix_indicator = suffix_indicator
209         self._oov_token = oov_token
210 
211         self._fast_word_piece = tf_text.FastWordpieceTokenizer(
212             vocab=self._vocab,
213             token_out_type=self.compute_dtype,
214             suffix_indicator=suffix_indicator,
215             unknown_token=oov_token,
216             no_pretokenization=True,
217             support_detokenization=True,
218         )
219 
220     def get_vocabulary(self) -> List[str]:
221         """Get the tokenizer vocabulary as a list of strings tokens."""
222         return self._vocab
223 
224     def vocabulary_size(self) -> int:
225         """Get the size of the tokenizer vocabulary."""
226         return len(self._vocab)
227 
228     def id_to_token(self, id: int) -> str:
229         """Convert an integer id to a string token."""
230         return self._vocab[id]
231 
232     def token_to_id(self, token: str) -> int:
233         """Convert a string token to an integer id."""
234         # This will be slow, but keep memory usage down compared to building a
235         # dict. Assuming the main use case is looking up a few special tokens
236         # early in the vocab, this should be fine.
237         return self._vocab.index(token)
238 
239     def get_config(self) -> Dict[str, Any]:
240         config = super().get_config()
241         config.update(
242             {
243                 # Ideally a vocabulary would be saved as a plain text asset in
244                 # the saved model. We have no good way to support this
245                 # currently, so we save the vocabulary in the config.
246                 "vocabulary": self._vocab,
247                 "sequence_length": self._sequence_length,
248                 "lowercase": self._lowercase,
249                 "strip_accents": self._strip_accents,
250                 "split_pattern": self._split_pattern,
251                 "keep_pattern": self._keep_pattern,
252                 "suffix_indicator": self._suffix_indicator,
253                 "oov_token": self._oov_token,
254             }
255         )
256         return config
257 
258     def tokenize(self, inputs):
259         # Optionally normalize and split inputs.
260         if self._lowercase:
261             inputs = tf_text.case_fold_utf8(inputs)
262         if self._strip_accents:
263             # Normalize unicode to NFD, which splits out accent mark characters.
264             inputs = tf_text.normalize_utf8(inputs, "NFD")
265             # Remove the accent marks.
266             inputs = tf.strings.regex_replace(inputs, r"\p{Mn}", "")
267         if self._split_pattern:
268             inputs = tf_text.regex_split(
269                 inputs,
270                 delim_regex_pattern=self._split_pattern,
271                 keep_delim_regex_pattern=self._keep_pattern,
272             )
273 
274         # Apply word piece and coerce shape for outputs.
275         tokens = self._fast_word_piece.tokenize(inputs)
276         # By default tf.text tokenizes text with two ragged dimensions (one for
277         # split words and one for split subwords). We will collapse to a single
278         # ragged dimension which is a better out of box default.
279         tokens = tokens.merge_dims(-2, -1)
280         # Convert to a dense output if `sequence_length` is set.
281         if self._sequence_length:
282             output_shape = tokens.shape.as_list()
283             output_shape[-1] = self._sequence_length
284             tokens = tokens.to_tensor(shape=output_shape)
285         return tokens
286 
287     def detokenize(self, inputs):
288         return self._fast_word_piece.detokenize(inputs)
289
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/keras_nlp/tokenizers/word_piece_tokenizer.py b/keras_nlp/tokenizers/word_piece_tokenizer.py
--- a/keras_nlp/tokenizers/word_piece_tokenizer.py
+++ b/keras_nlp/tokenizers/word_piece_tokenizer.py
@@ -256,6 +256,12 @@
         return config
 
     def tokenize(self, inputs):
+        # Check if Input is Scalar or Not
+        if not isinstance(inputs, (tf.Tensor, tf.RaggedTensor)):
+            inputs = tf.convert_to_tensor(inputs)
+        scalar_input = tf.convert_to_tensor(inputs).shape.rank == 0
+        if scalar_input:
+            inputs = tf.expand_dims(inputs, 0)
         # Optionally normalize and split inputs.
         if self._lowercase:
             inputs = tf_text.case_fold_utf8(inputs)
@@ -282,6 +288,11 @@
             output_shape = tokens.shape.as_list()
             output_shape[-1] = self._sequence_length
             tokens = tokens.to_tensor(shape=output_shape)
+        # Convert to a dense output if input in scalar
+        if scalar_input:
+            tokens = tf.squeeze(tokens, 0)
+            tf.ensure_shape(tokens, shape=[self._sequence_length])
+
         return tokens
 
     def detokenize(self, inputs):

{"golden_diff": "diff --git a/keras_nlp/tokenizers/word_piece_tokenizer.py b/keras_nlp/tokenizers/word_piece_tokenizer.py\n--- a/keras_nlp/tokenizers/word_piece_tokenizer.py\n+++ b/keras_nlp/tokenizers/word_piece_tokenizer.py\n@@ -256,6 +256,12 @@\n         return config\n \n     def tokenize(self, inputs):\n+        # Check if Input is Scalar or Not\n+        if not isinstance(inputs, (tf.Tensor, tf.RaggedTensor)):\n+            inputs = tf.convert_to_tensor(inputs)\n+        scalar_input = tf.convert_to_tensor(inputs).shape.rank == 0\n+        if scalar_input:\n+            inputs = tf.expand_dims(inputs, 0)\n         # Optionally normalize and split inputs.\n         if self._lowercase:\n             inputs = tf_text.case_fold_utf8(inputs)\n@@ -282,6 +288,11 @@\n             output_shape = tokens.shape.as_list()\n             output_shape[-1] = self._sequence_length\n             tokens = tokens.to_tensor(shape=output_shape)\n+        # Convert to a dense output if input in scalar\n+        if scalar_input:\n+            tokens = tf.squeeze(tokens, 0)\n+            tf.ensure_shape(tokens, shape=[self._sequence_length])\n+\n         return tokens\n \n     def detokenize(self, inputs):\n", "issue": "Fix rank 1 outputs for WordPieceTokenizer\nRight now, a scalar string inputs to the tokenizer will turn into rank 2 outputs, which we do not want. Scalar inputs should result in rank 1 outputs.\r\n\r\nSee this colab for a reproduction of the issue:\r\nhttps://colab.research.google.com/gist/mattdangerw/8e1fdbebadf0ff6ff2dc065b2e00db65/rank1wordpiece.ipynb\n", "before_files": [{"content": "# Copyright 2022 The KerasNLP Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Any\nfrom typing import Dict\nfrom typing import Iterable\nfrom typing import List\nfrom typing import Union\n\nimport tensorflow as tf\nimport tensorflow_text as tf_text\n\nfrom keras_nlp.tokenizers import tokenizer\n\n# Matches whitespace and control characters.\nWHITESPACE_REGEX = r\"|\".join(\n    [\n        r\"\\s\",\n        # Invisible control characters\n        r\"\\p{Cc}\",\n        r\"\\p{Cf}\",\n    ]\n)\n\n# Matches punctuation compatible with the original bert implementation.\nPUNCTUATION_REGEX = r\"|\".join(\n    [\n        # Treat all non-letter/number ASCII as punctuation.\n        # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n        # Punctuation class but we treat them as punctuation anyways.\n        r\"[!-/]\",\n        r\"[:-@]\",\n        r\"[\\[-`]\",\n        r\"[{-~]\",\n        # Unicode punctuation class.\n        r\"[\\p{P}]\",\n        # More unicode ranges.\n        r\"[\\x{4E00}-\\x{9FFF}]\",\n        r\"[\\x{3400}-\\x{4DBF}]\",\n        r\"[\\x{20000}-\\x{2A6DF}]\",\n        r\"[\\x{2A700}-\\x{2B73F}]\",\n        r\"[\\x{2B740}-\\x{2B81F}]\",\n        r\"[\\x{2B820}-\\x{2CEAF}]\",\n        r\"[\\x{F900}-\\x{FAFF}]\",\n        r\"[\\x{2F800}-\\x{2FA1F}]\",\n    ]\n)\n\n# Matches both whitespace and punctuation.\nWHITESPACE_AND_PUNCTUATION_REGEX = r\"|\".join(\n    [\n        WHITESPACE_REGEX,\n        PUNCTUATION_REGEX,\n    ]\n)\n\n\nclass WordPieceTokenizer(tokenizer.Tokenizer):\n    \"\"\"A word piece tokenizer layer.\n\n    This layer provides an efficient, in graph, implementation of the WordPiece\n    algorithm used by BERT and other models.\n\n    To make this layer more useful out of the box, the layer will pre-tokenize\n    the input, which will optionally lower-case, strip accents, and split the\n    input on whitespace and punctuation. Each of these pre-tokenization steps is\n    not reversible. The `detokenize` method will join words with a space, and\n    will not invert `tokenize` exactly.\n\n    If a more custom pre-tokenization step is desired, the layer can be\n    configured to apply only the strict WordPiece algorithm by passing\n    `lowercase=False`, `strip_accents=False` and `split_pattern=None`. In\n    this case, inputs should be pre-split string tensors or ragged tensors.\n\n    By default, the layer will output a `tf.RaggedTensor` where the last\n    dimension of the output is ragged after whitespace splitting and sub-word\n    tokenizing. If `sequence_length` is set, the layer will output a dense\n    `tf.Tensor` where all inputs have been padded or truncated to\n    `sequence_length`. The output dtype can be controlled via the `dtype`\n    argument, which should be either an integer or string type.\n\n    Args:\n        vocabulary: A list of strings or a string string filename path. If\n            passing a list, each element of the list should be a single word\n            piece token string. If passing a filename, the file should be a\n            plain text file containing a single word piece token per line.\n        sequence_length: If set, the output will be converted to a dense\n            tensor and padded/trimmed so all outputs are of sequence_length.\n        lowercase: If true, the input text will be first lowered before\n            tokenization.\n        strip_accents: If true, all accent marks will be removed from text\n            before tokenization.\n        split_pattern: A regex pattern to match delimiters to split, or None\n            indicating that the input is pre-split and no splitting should be\n            performed. By default, all whitespace and punctuation marks will\n            be split on.\n        keep_pattern: A regex pattern of delimiters contained in the\n            `split_pattern` of delimeters that should be kept as independent\n            tokens. By default, all punctuation marks will be kept as tokens.\n        suffix_indicator: The characters prepended to a wordpiece to indicate\n            that it is a suffix to another subword.\n        oov_token: The string value to substitute for an unknown token. It\n            must be included in the vocab.\n\n    References:\n        - [Schuster and Nakajima, 2012](https://research.google/pubs/pub37842/)\n        - [Song et al., 2020](https://arxiv.org/abs/2012.15524)\n\n    Examples:\n\n    Ragged outputs.\n    >>> vocab = [\"[UNK]\", \"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\", \".\"]\n    >>> inputs = [\"The quick brown fox.\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab)\n    >>> tokenizer(inputs)\n    <tf.RaggedTensor [[1, 2, 3, 4, 5, 6, 7]]>\n\n    Dense outputs.\n    >>> vocab = [\"[UNK]\", \"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\", \".\"]\n    >>> inputs = [\"The quick brown fox.\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(\n    ...     vocabulary=vocab, sequence_length=10)\n    >>> tokenizer(inputs)\n    <tf.Tensor: shape=(1, 10), dtype=int32,\n        numpy=array([[1, 2, 3, 4, 5, 6, 7, 0, 0, 0]]>\n\n    String output.\n    >>> vocab = [\"[UNK]\", \"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\", \".\"]\n    >>> inputs = [\"The quick brown fox.\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(\n    ...     vocabulary=vocab, dtype=\"string\")\n    >>> tokenizer(inputs)\n    <tf.RaggedTensor [[b'the', b'qu', b'##ick', b'br', b'##own', b'fox', b'.']]>\n\n    Detokenization.\n    >>> vocab = [\"[UNK]\", \"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\", \".\"]\n    >>> inputs = [\"The quick brown fox.\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab)\n    >>> tokenizer.detokenize(tokenizer.tokenize(inputs))\n    <tf.Tensor: shape=(1,), dtype=string,\n        numpy=array([b\"the quick brown fox .\"], dtype=object)>\n\n    Custom splitting.\n    >>> vocab = [\"[UNK]\", \"fox\", \",\"]\n    >>> inputs = [\"fox,,fox,fox\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab,\n    ...     split_pattern=\",\", keep_pattern=\",\", dtype='string')(inputs)\n    <tf.RaggedTensor [[b'fox', b',', b',', b'fox', b',', b'fox']]>\n    >>> keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab,\n    ...     split_pattern=\",\", keep_pattern=\"\", dtype='string')(inputs)\n    <tf.RaggedTensor [[b'fox', b'fox', b'fox']]>\n    \"\"\"\n\n    def __init__(\n        self,\n        vocabulary: Union[Iterable[str], str] = None,\n        sequence_length: int = None,\n        lowercase: bool = True,\n        strip_accents: bool = True,\n        split_pattern: str = WHITESPACE_AND_PUNCTUATION_REGEX,\n        keep_pattern: str = PUNCTUATION_REGEX,\n        suffix_indicator: str = \"##\",\n        oov_token: str = \"[UNK]\",\n        **kwargs,\n    ) -> None:\n        # Check dtype and provide a default.\n        if \"dtype\" not in kwargs or kwargs[\"dtype\"] is None:\n            kwargs[\"dtype\"] = tf.int32\n        else:\n            dtype = tf.dtypes.as_dtype(kwargs[\"dtype\"])\n            if not dtype.is_integer and dtype != tf.string:\n                raise ValueError(\n                    \"Output dtype must be an integer type of a string. \"\n                    f\"Received: dtype={dtype}\"\n                )\n\n        super().__init__(**kwargs)\n\n        if isinstance(vocabulary, str):\n            self._vocab = [\n                line.rstrip() for line in tf.io.gfile.GFile(vocabulary)\n            ]\n        elif isinstance(vocabulary, Iterable):\n            # Make a copy.\n            self._vocab = list(vocabulary)\n        else:\n            raise ValueError(\n                \"Vocabulary must be an file path or list of terms. \"\n                f\"Received: vocabulary={vocabulary}\"\n            )\n\n        self._sequence_length = sequence_length\n        self._lowercase = lowercase\n        self._strip_accents = strip_accents\n        self._split_pattern = split_pattern\n        self._keep_pattern = keep_pattern\n        self._suffix_indicator = suffix_indicator\n        self._oov_token = oov_token\n\n        self._fast_word_piece = tf_text.FastWordpieceTokenizer(\n            vocab=self._vocab,\n            token_out_type=self.compute_dtype,\n            suffix_indicator=suffix_indicator,\n            unknown_token=oov_token,\n            no_pretokenization=True,\n            support_detokenization=True,\n        )\n\n    def get_vocabulary(self) -> List[str]:\n        \"\"\"Get the tokenizer vocabulary as a list of strings tokens.\"\"\"\n        return self._vocab\n\n    def vocabulary_size(self) -> int:\n        \"\"\"Get the size of the tokenizer vocabulary.\"\"\"\n        return len(self._vocab)\n\n    def id_to_token(self, id: int) -> str:\n        \"\"\"Convert an integer id to a string token.\"\"\"\n        return self._vocab[id]\n\n    def token_to_id(self, token: str) -> int:\n        \"\"\"Convert a string token to an integer id.\"\"\"\n        # This will be slow, but keep memory usage down compared to building a\n        # dict. Assuming the main use case is looking up a few special tokens\n        # early in the vocab, this should be fine.\n        return self._vocab.index(token)\n\n    def get_config(self) -> Dict[str, Any]:\n        config = super().get_config()\n        config.update(\n            {\n                # Ideally a vocabulary would be saved as a plain text asset in\n                # the saved model. We have no good way to support this\n                # currently, so we save the vocabulary in the config.\n                \"vocabulary\": self._vocab,\n                \"sequence_length\": self._sequence_length,\n                \"lowercase\": self._lowercase,\n                \"strip_accents\": self._strip_accents,\n                \"split_pattern\": self._split_pattern,\n                \"keep_pattern\": self._keep_pattern,\n                \"suffix_indicator\": self._suffix_indicator,\n                \"oov_token\": self._oov_token,\n            }\n        )\n        return config\n\n    def tokenize(self, inputs):\n        # Optionally normalize and split inputs.\n        if self._lowercase:\n            inputs = tf_text.case_fold_utf8(inputs)\n        if self._strip_accents:\n            # Normalize unicode to NFD, which splits out accent mark characters.\n            inputs = tf_text.normalize_utf8(inputs, \"NFD\")\n            # Remove the accent marks.\n            inputs = tf.strings.regex_replace(inputs, r\"\\p{Mn}\", \"\")\n        if self._split_pattern:\n            inputs = tf_text.regex_split(\n                inputs,\n                delim_regex_pattern=self._split_pattern,\n                keep_delim_regex_pattern=self._keep_pattern,\n            )\n\n        # Apply word piece and coerce shape for outputs.\n        tokens = self._fast_word_piece.tokenize(inputs)\n        # By default tf.text tokenizes text with two ragged dimensions (one for\n        # split words and one for split subwords). We will collapse to a single\n        # ragged dimension which is a better out of box default.\n        tokens = tokens.merge_dims(-2, -1)\n        # Convert to a dense output if `sequence_length` is set.\n        if self._sequence_length:\n            output_shape = tokens.shape.as_list()\n            output_shape[-1] = self._sequence_length\n            tokens = tokens.to_tensor(shape=output_shape)\n        return tokens\n\n    def detokenize(self, inputs):\n        return self._fast_word_piece.detokenize(inputs)\n", "path": "keras_nlp/tokenizers/word_piece_tokenizer.py"}], "after_files": [{"content": "# Copyright 2022 The KerasNLP Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Any\nfrom typing import Dict\nfrom typing import Iterable\nfrom typing import List\nfrom typing import Union\n\nimport tensorflow as tf\nimport tensorflow_text as tf_text\n\nfrom keras_nlp.tokenizers import tokenizer\n\n# Matches whitespace and control characters.\nWHITESPACE_REGEX = r\"|\".join(\n    [\n        r\"\\s\",\n        # Invisible control characters\n        r\"\\p{Cc}\",\n        r\"\\p{Cf}\",\n    ]\n)\n\n# Matches punctuation compatible with the original bert implementation.\nPUNCTUATION_REGEX = r\"|\".join(\n    [\n        # Treat all non-letter/number ASCII as punctuation.\n        # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n        # Punctuation class but we treat them as punctuation anyways.\n        r\"[!-/]\",\n        r\"[:-@]\",\n        r\"[\\[-`]\",\n        r\"[{-~]\",\n        # Unicode punctuation class.\n        r\"[\\p{P}]\",\n        # More unicode ranges.\n        r\"[\\x{4E00}-\\x{9FFF}]\",\n        r\"[\\x{3400}-\\x{4DBF}]\",\n        r\"[\\x{20000}-\\x{2A6DF}]\",\n        r\"[\\x{2A700}-\\x{2B73F}]\",\n        r\"[\\x{2B740}-\\x{2B81F}]\",\n        r\"[\\x{2B820}-\\x{2CEAF}]\",\n        r\"[\\x{F900}-\\x{FAFF}]\",\n        r\"[\\x{2F800}-\\x{2FA1F}]\",\n    ]\n)\n\n# Matches both whitespace and punctuation.\nWHITESPACE_AND_PUNCTUATION_REGEX = r\"|\".join(\n    [\n        WHITESPACE_REGEX,\n        PUNCTUATION_REGEX,\n    ]\n)\n\n\nclass WordPieceTokenizer(tokenizer.Tokenizer):\n    \"\"\"A word piece tokenizer layer.\n\n    This layer provides an efficient, in graph, implementation of the WordPiece\n    algorithm used by BERT and other models.\n\n    To make this layer more useful out of the box, the layer will pre-tokenize\n    the input, which will optionally lower-case, strip accents, and split the\n    input on whitespace and punctuation. Each of these pre-tokenization steps is\n    not reversible. The `detokenize` method will join words with a space, and\n    will not invert `tokenize` exactly.\n\n    If a more custom pre-tokenization step is desired, the layer can be\n    configured to apply only the strict WordPiece algorithm by passing\n    `lowercase=False`, `strip_accents=False` and `split_pattern=None`. In\n    this case, inputs should be pre-split string tensors or ragged tensors.\n\n    By default, the layer will output a `tf.RaggedTensor` where the last\n    dimension of the output is ragged after whitespace splitting and sub-word\n    tokenizing. If `sequence_length` is set, the layer will output a dense\n    `tf.Tensor` where all inputs have been padded or truncated to\n    `sequence_length`. The output dtype can be controlled via the `dtype`\n    argument, which should be either an integer or string type.\n\n    Args:\n        vocabulary: A list of strings or a string string filename path. If\n            passing a list, each element of the list should be a single word\n            piece token string. If passing a filename, the file should be a\n            plain text file containing a single word piece token per line.\n        sequence_length: If set, the output will be converted to a dense\n            tensor and padded/trimmed so all outputs are of sequence_length.\n        lowercase: If true, the input text will be first lowered before\n            tokenization.\n        strip_accents: If true, all accent marks will be removed from text\n            before tokenization.\n        split_pattern: A regex pattern to match delimiters to split, or None\n            indicating that the input is pre-split and no splitting should be\n            performed. By default, all whitespace and punctuation marks will\n            be split on.\n        keep_pattern: A regex pattern of delimiters contained in the\n            `split_pattern` of delimeters that should be kept as independent\n            tokens. By default, all punctuation marks will be kept as tokens.\n        suffix_indicator: The characters prepended to a wordpiece to indicate\n            that it is a suffix to another subword.\n        oov_token: The string value to substitute for an unknown token. It\n            must be included in the vocab.\n\n    References:\n        - [Schuster and Nakajima, 2012](https://research.google/pubs/pub37842/)\n        - [Song et al., 2020](https://arxiv.org/abs/2012.15524)\n\n    Examples:\n\n    Ragged outputs.\n    >>> vocab = [\"[UNK]\", \"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\", \".\"]\n    >>> inputs = [\"The quick brown fox.\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab)\n    >>> tokenizer(inputs)\n    <tf.RaggedTensor [[1, 2, 3, 4, 5, 6, 7]]>\n\n    Dense outputs.\n    >>> vocab = [\"[UNK]\", \"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\", \".\"]\n    >>> inputs = [\"The quick brown fox.\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(\n    ...     vocabulary=vocab, sequence_length=10)\n    >>> tokenizer(inputs)\n    <tf.Tensor: shape=(1, 10), dtype=int32,\n        numpy=array([[1, 2, 3, 4, 5, 6, 7, 0, 0, 0]]>\n\n    String output.\n    >>> vocab = [\"[UNK]\", \"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\", \".\"]\n    >>> inputs = [\"The quick brown fox.\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(\n    ...     vocabulary=vocab, dtype=\"string\")\n    >>> tokenizer(inputs)\n    <tf.RaggedTensor [[b'the', b'qu', b'##ick', b'br', b'##own', b'fox', b'.']]>\n\n    Detokenization.\n    >>> vocab = [\"[UNK]\", \"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\", \".\"]\n    >>> inputs = [\"The quick brown fox.\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab)\n    >>> tokenizer.detokenize(tokenizer.tokenize(inputs))\n    <tf.Tensor: shape=(1,), dtype=string,\n        numpy=array([b\"the quick brown fox .\"], dtype=object)>\n\n    Custom splitting.\n    >>> vocab = [\"[UNK]\", \"fox\", \",\"]\n    >>> inputs = [\"fox,,fox,fox\"]\n    >>> tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab,\n    ...     split_pattern=\",\", keep_pattern=\",\", dtype='string')(inputs)\n    <tf.RaggedTensor [[b'fox', b',', b',', b'fox', b',', b'fox']]>\n    >>> keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab,\n    ...     split_pattern=\",\", keep_pattern=\"\", dtype='string')(inputs)\n    <tf.RaggedTensor [[b'fox', b'fox', b'fox']]>\n    \"\"\"\n\n    def __init__(\n        self,\n        vocabulary: Union[Iterable[str], str] = None,\n        sequence_length: int = None,\n        lowercase: bool = True,\n        strip_accents: bool = True,\n        split_pattern: str = WHITESPACE_AND_PUNCTUATION_REGEX,\n        keep_pattern: str = PUNCTUATION_REGEX,\n        suffix_indicator: str = \"##\",\n        oov_token: str = \"[UNK]\",\n        **kwargs,\n    ) -> None:\n        # Check dtype and provide a default.\n        if \"dtype\" not in kwargs or kwargs[\"dtype\"] is None:\n            kwargs[\"dtype\"] = tf.int32\n        else:\n            dtype = tf.dtypes.as_dtype(kwargs[\"dtype\"])\n            if not dtype.is_integer and dtype != tf.string:\n                raise ValueError(\n                    \"Output dtype must be an integer type of a string. \"\n                    f\"Received: dtype={dtype}\"\n                )\n\n        super().__init__(**kwargs)\n\n        if isinstance(vocabulary, str):\n            self._vocab = [\n                line.rstrip() for line in tf.io.gfile.GFile(vocabulary)\n            ]\n        elif isinstance(vocabulary, Iterable):\n            # Make a copy.\n            self._vocab = list(vocabulary)\n        else:\n            raise ValueError(\n                \"Vocabulary must be an file path or list of terms. \"\n                f\"Received: vocabulary={vocabulary}\"\n            )\n\n        self._sequence_length = sequence_length\n        self._lowercase = lowercase\n        self._strip_accents = strip_accents\n        self._split_pattern = split_pattern\n        self._keep_pattern = keep_pattern\n        self._suffix_indicator = suffix_indicator\n        self._oov_token = oov_token\n\n        self._fast_word_piece = tf_text.FastWordpieceTokenizer(\n            vocab=self._vocab,\n            token_out_type=self.compute_dtype,\n            suffix_indicator=suffix_indicator,\n            unknown_token=oov_token,\n            no_pretokenization=True,\n            support_detokenization=True,\n        )\n\n    def get_vocabulary(self) -> List[str]:\n        \"\"\"Get the tokenizer vocabulary as a list of strings tokens.\"\"\"\n        return self._vocab\n\n    def vocabulary_size(self) -> int:\n        \"\"\"Get the size of the tokenizer vocabulary.\"\"\"\n        return len(self._vocab)\n\n    def id_to_token(self, id: int) -> str:\n        \"\"\"Convert an integer id to a string token.\"\"\"\n        return self._vocab[id]\n\n    def token_to_id(self, token: str) -> int:\n        \"\"\"Convert a string token to an integer id.\"\"\"\n        # This will be slow, but keep memory usage down compared to building a\n        # dict. Assuming the main use case is looking up a few special tokens\n        # early in the vocab, this should be fine.\n        return self._vocab.index(token)\n\n    def get_config(self) -> Dict[str, Any]:\n        config = super().get_config()\n        config.update(\n            {\n                # Ideally a vocabulary would be saved as a plain text asset in\n                # the saved model. We have no good way to support this\n                # currently, so we save the vocabulary in the config.\n                \"vocabulary\": self._vocab,\n                \"sequence_length\": self._sequence_length,\n                \"lowercase\": self._lowercase,\n                \"strip_accents\": self._strip_accents,\n                \"split_pattern\": self._split_pattern,\n                \"keep_pattern\": self._keep_pattern,\n                \"suffix_indicator\": self._suffix_indicator,\n                \"oov_token\": self._oov_token,\n            }\n        )\n        return config\n\n    def tokenize(self, inputs):\n        # Check if Input is Scalar or Not\n        if not isinstance(inputs, (tf.Tensor, tf.RaggedTensor)):\n            inputs = tf.convert_to_tensor(inputs)\n        scalar_input = tf.convert_to_tensor(inputs).shape.rank == 0\n        if scalar_input:\n            inputs = tf.expand_dims(inputs, 0)\n        # Optionally normalize and split inputs.\n        if self._lowercase:\n            inputs = tf_text.case_fold_utf8(inputs)\n        if self._strip_accents:\n            # Normalize unicode to NFD, which splits out accent mark characters.\n            inputs = tf_text.normalize_utf8(inputs, \"NFD\")\n            # Remove the accent marks.\n            inputs = tf.strings.regex_replace(inputs, r\"\\p{Mn}\", \"\")\n        if self._split_pattern:\n            inputs = tf_text.regex_split(\n                inputs,\n                delim_regex_pattern=self._split_pattern,\n                keep_delim_regex_pattern=self._keep_pattern,\n            )\n\n        # Apply word piece and coerce shape for outputs.\n        tokens = self._fast_word_piece.tokenize(inputs)\n        # By default tf.text tokenizes text with two ragged dimensions (one for\n        # split words and one for split subwords). We will collapse to a single\n        # ragged dimension which is a better out of box default.\n        tokens = tokens.merge_dims(-2, -1)\n        # Convert to a dense output if `sequence_length` is set.\n        if self._sequence_length:\n            output_shape = tokens.shape.as_list()\n            output_shape[-1] = self._sequence_length\n            tokens = tokens.to_tensor(shape=output_shape)\n        # Convert to a dense output if input in scalar\n        if scalar_input:\n            tokens = tf.squeeze(tokens, 0)\n            tf.ensure_shape(tokens, shape=[self._sequence_length])\n\n        return tokens\n\n    def detokenize(self, inputs):\n        return self._fast_word_piece.detokenize(inputs)\n", "path": "keras_nlp/tokenizers/word_piece_tokenizer.py"}]}