We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Watermarking bug
### System Info

using singularity with the container `text-generation-inference:1.0.0` .

i get this error:

```
Traceback (most recent call last):
  File "/opt/conda/lib/python3.9/site-packages/text_generation_server/models/flash_causal_lm.py", line 727, in warmup
    _, batch = self.generate_token(batch)
  File "/opt/conda/lib/python3.9/contextlib.py", line 79, in inner
    return func(*args, **kwds)
  File "/opt/conda/lib/python3.9/site-packages/text_generation_server/models/flash_causal_lm.py", line 834, in generate_token
    next_input_ids, next_token_logprobs = batch.next_token_chooser(
  File "/opt/conda/lib/python3.9/site-packages/text_generation_server/utils/tokens.py", line 224, in __call__
    scores = self.watermark_processor(input_ids, scores)
  File "/opt/conda/lib/python3.9/site-packages/text_generation_server/utils/logits_process.py", line 398, in __call__
    scores[i : i + 1] = processor(input_ids[i : i + 1], scores[i : i + 1])
  File "/opt/conda/lib/python3.9/site-packages/text_generation_server/utils/watermark.py", line 88, in __call__
    greenlist_ids = self._get_greenlist_ids(
  File "/opt/conda/lib/python3.9/site-packages/text_generation_server/utils/watermark.py", line 64, in _get_greenlist_ids
    greenlist_size = int(max_value * self.gamma)
ValueError: invalid literal for int() with base 10: '0.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50
```

i think `greenlist_size = int(max_value * self.gamma)` should be `greenlist_size = max_value * int(self.gamma)`

I also tried setting the env variables only `export WATERMARK_GAMMA=0.5 && export WATERMARK_DELTA=2.0` and the exact issue.


### Information

- [X] Docker
- [ ] The CLI directly

### Tasks

- [X] An officially supported command
- [ ] My own modifications

### Reproduction

```bash
export USE_FLASH_ATTENTION=True && text-generation-launcher \
--model-id /scratch//models/huggyllama-llama-7b \
--port 61697 \
--max-best-of 5 \
--max-total-tokens 2048 \
--max-input-length 1024 \
--max-batch-prefill-tokens 1024 \
--max-concurrent-requests 20 \
--sharded false \
--num-shard 1  \
--dtype float16 \
 --disable-custom-kernels  \
--watermark-gamma 0.5 \
--watermark-delta 2
```

### Expected behavior

It should multiply the number not the string `0.5`

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `server/text_generation_server/utils/watermark.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2023 Authors of "A Watermark for Large Language Models"
3 # available at https://arxiv.org/abs/2301.10226
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 import os
17 
18 import torch
19 from transformers import LogitsProcessor
20 from typing import List, Union
21 
22 GAMMA = os.getenv("WATERMARK_GAMMA", 0.5)
23 DELTA = os.getenv("WATERMARK_DELTA", 2.0)
24 
25 
26 class WatermarkLogitsProcessor(LogitsProcessor):
27     def __init__(
28         self,
29         gamma: float = GAMMA,
30         delta: float = DELTA,
31         hash_key: int = 15485863,  # just a large prime number to create a rng seed with sufficient bit width
32         device: str = "cpu",
33     ):
34         # watermarking parameters
35         self.gamma = gamma
36         self.delta = delta
37         self.rng = torch.Generator(device=device)
38         self.hash_key = hash_key
39 
40     def _seed_rng(self, input_ids: Union[List[int], torch.LongTensor]):
41         if isinstance(input_ids, list):
42             assert (
43                 len(input_ids) >= 1
44             ), "requires at least a 1 token prefix sequence to seed rng"
45             prev_token = input_ids[-1]
46         else:
47             assert len(input_ids) == 1
48             input_ids = input_ids[0]
49             assert (
50                 input_ids.shape[-1] >= 1
51             ), "requires at least a 1 token prefix sequence to seed rng"
52             prev_token = input_ids[-1].item()
53         self.rng.manual_seed(self.hash_key * prev_token)
54 
55     def _get_greenlist_ids(
56         self,
57         input_ids: Union[List[int], torch.LongTensor],
58         max_value: int,
59         device: torch.device,
60     ) -> List[int]:
61         # seed the rng using the previous tokens/prefix
62         self._seed_rng(input_ids)
63 
64         greenlist_size = int(max_value * self.gamma)
65         vocab_permutation = torch.randperm(max_value, device=device, generator=self.rng)
66         greenlist_ids = vocab_permutation[:greenlist_size]
67         return greenlist_ids
68 
69     @staticmethod
70     def _calc_greenlist_mask(
71         scores: torch.FloatTensor, greenlist_token_ids
72     ) -> torch.BoolTensor:
73         green_tokens_mask = torch.zeros_like(scores)
74         green_tokens_mask[-1, greenlist_token_ids] = 1
75         final_mask = green_tokens_mask.bool()
76         return final_mask
77 
78     @staticmethod
79     def _bias_greenlist_logits(
80         scores: torch.Tensor, greenlist_mask: torch.Tensor, greenlist_bias: float
81     ) -> torch.Tensor:
82         scores[greenlist_mask] = scores[greenlist_mask] + greenlist_bias
83         return scores
84 
85     def __call__(
86         self, input_ids: Union[List[int], torch.LongTensor], scores: torch.FloatTensor
87     ) -> torch.FloatTensor:
88         greenlist_ids = self._get_greenlist_ids(
89             input_ids, scores.shape[-1], scores.device
90         )
91         green_tokens_mask = self._calc_greenlist_mask(
92             scores=scores, greenlist_token_ids=greenlist_ids
93         )
94 
95         scores = self._bias_greenlist_logits(
96             scores=scores, greenlist_mask=green_tokens_mask, greenlist_bias=self.delta
97         )
98         return scores
99
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/server/text_generation_server/utils/watermark.py b/server/text_generation_server/utils/watermark.py
--- a/server/text_generation_server/utils/watermark.py
+++ b/server/text_generation_server/utils/watermark.py
@@ -19,8 +19,8 @@
 from transformers import LogitsProcessor
 from typing import List, Union
 
-GAMMA = os.getenv("WATERMARK_GAMMA", 0.5)
-DELTA = os.getenv("WATERMARK_DELTA", 2.0)
+GAMMA = float(os.getenv("WATERMARK_GAMMA", 0.5))
+DELTA = float(os.getenv("WATERMARK_DELTA", 2.0))
 
 
 class WatermarkLogitsProcessor(LogitsProcessor):

{"golden_diff": "diff --git a/server/text_generation_server/utils/watermark.py b/server/text_generation_server/utils/watermark.py\n--- a/server/text_generation_server/utils/watermark.py\n+++ b/server/text_generation_server/utils/watermark.py\n@@ -19,8 +19,8 @@\n from transformers import LogitsProcessor\n from typing import List, Union\n \n-GAMMA = os.getenv(\"WATERMARK_GAMMA\", 0.5)\n-DELTA = os.getenv(\"WATERMARK_DELTA\", 2.0)\n+GAMMA = float(os.getenv(\"WATERMARK_GAMMA\", 0.5))\n+DELTA = float(os.getenv(\"WATERMARK_DELTA\", 2.0))\n \n \n class WatermarkLogitsProcessor(LogitsProcessor):\n", "issue": "Watermarking bug\n### System Info\n\nusing singularity with the container `text-generation-inference:1.0.0` .\r\n\r\ni get this error:\r\n\r\n```\r\nTraceback (most recent call last):\r\n  File \"/opt/conda/lib/python3.9/site-packages/text_generation_server/models/flash_causal_lm.py\", line 727, in warmup\r\n    _, batch = self.generate_token(batch)\r\n  File \"/opt/conda/lib/python3.9/contextlib.py\", line 79, in inner\r\n    return func(*args, **kwds)\r\n  File \"/opt/conda/lib/python3.9/site-packages/text_generation_server/models/flash_causal_lm.py\", line 834, in generate_token\r\n    next_input_ids, next_token_logprobs = batch.next_token_chooser(\r\n  File \"/opt/conda/lib/python3.9/site-packages/text_generation_server/utils/tokens.py\", line 224, in __call__\r\n    scores = self.watermark_processor(input_ids, scores)\r\n  File \"/opt/conda/lib/python3.9/site-packages/text_generation_server/utils/logits_process.py\", line 398, in __call__\r\n    scores[i : i + 1] = processor(input_ids[i : i + 1], scores[i : i + 1])\r\n  File \"/opt/conda/lib/python3.9/site-packages/text_generation_server/utils/watermark.py\", line 88, in __call__\r\n    greenlist_ids = self._get_greenlist_ids(\r\n  File \"/opt/conda/lib/python3.9/site-packages/text_generation_server/utils/watermark.py\", line 64, in _get_greenlist_ids\r\n    greenlist_size = int(max_value * self.gamma)\r\nValueError: invalid literal for int() with base 10: '0.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50.50\r\n```\r\n\r\ni think `greenlist_size = int(max_value * self.gamma)` should be `greenlist_size = max_value * int(self.gamma)`\r\n\r\nI also tried setting the env variables only `export WATERMARK_GAMMA=0.5 && export WATERMARK_DELTA=2.0` and the exact issue.\r\n\n\n### Information\n\n- [X] Docker\n- [ ] The CLI directly\n\n### Tasks\n\n- [X] An officially supported command\n- [ ] My own modifications\n\n### Reproduction\n\n```bash\r\nexport USE_FLASH_ATTENTION=True && text-generation-launcher \\\r\n--model-id /scratch//models/huggyllama-llama-7b \\\r\n--port 61697 \\\r\n--max-best-of 5 \\\r\n--max-total-tokens 2048 \\\r\n--max-input-length 1024 \\\r\n--max-batch-prefill-tokens 1024 \\\r\n--max-concurrent-requests 20 \\\r\n--sharded false \\\r\n--num-shard 1  \\\r\n--dtype float16 \\\r\n --disable-custom-kernels  \\\r\n--watermark-gamma 0.5 \\\r\n--watermark-delta 2\r\n```\n\n### Expected behavior\n\nIt should multiply the number not the string `0.5`\n", "before_files": [{"content": "# coding=utf-8\n# Copyright 2023 Authors of \"A Watermark for Large Language Models\"\n# available at https://arxiv.org/abs/2301.10226\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport os\n\nimport torch\nfrom transformers import LogitsProcessor\nfrom typing import List, Union\n\nGAMMA = os.getenv(\"WATERMARK_GAMMA\", 0.5)\nDELTA = os.getenv(\"WATERMARK_DELTA\", 2.0)\n\n\nclass WatermarkLogitsProcessor(LogitsProcessor):\n    def __init__(\n        self,\n        gamma: float = GAMMA,\n        delta: float = DELTA,\n        hash_key: int = 15485863,  # just a large prime number to create a rng seed with sufficient bit width\n        device: str = \"cpu\",\n    ):\n        # watermarking parameters\n        self.gamma = gamma\n        self.delta = delta\n        self.rng = torch.Generator(device=device)\n        self.hash_key = hash_key\n\n    def _seed_rng(self, input_ids: Union[List[int], torch.LongTensor]):\n        if isinstance(input_ids, list):\n            assert (\n                len(input_ids) >= 1\n            ), \"requires at least a 1 token prefix sequence to seed rng\"\n            prev_token = input_ids[-1]\n        else:\n            assert len(input_ids) == 1\n            input_ids = input_ids[0]\n            assert (\n                input_ids.shape[-1] >= 1\n            ), \"requires at least a 1 token prefix sequence to seed rng\"\n            prev_token = input_ids[-1].item()\n        self.rng.manual_seed(self.hash_key * prev_token)\n\n    def _get_greenlist_ids(\n        self,\n        input_ids: Union[List[int], torch.LongTensor],\n        max_value: int,\n        device: torch.device,\n    ) -> List[int]:\n        # seed the rng using the previous tokens/prefix\n        self._seed_rng(input_ids)\n\n        greenlist_size = int(max_value * self.gamma)\n        vocab_permutation = torch.randperm(max_value, device=device, generator=self.rng)\n        greenlist_ids = vocab_permutation[:greenlist_size]\n        return greenlist_ids\n\n    @staticmethod\n    def _calc_greenlist_mask(\n        scores: torch.FloatTensor, greenlist_token_ids\n    ) -> torch.BoolTensor:\n        green_tokens_mask = torch.zeros_like(scores)\n        green_tokens_mask[-1, greenlist_token_ids] = 1\n        final_mask = green_tokens_mask.bool()\n        return final_mask\n\n    @staticmethod\n    def _bias_greenlist_logits(\n        scores: torch.Tensor, greenlist_mask: torch.Tensor, greenlist_bias: float\n    ) -> torch.Tensor:\n        scores[greenlist_mask] = scores[greenlist_mask] + greenlist_bias\n        return scores\n\n    def __call__(\n        self, input_ids: Union[List[int], torch.LongTensor], scores: torch.FloatTensor\n    ) -> torch.FloatTensor:\n        greenlist_ids = self._get_greenlist_ids(\n            input_ids, scores.shape[-1], scores.device\n        )\n        green_tokens_mask = self._calc_greenlist_mask(\n            scores=scores, greenlist_token_ids=greenlist_ids\n        )\n\n        scores = self._bias_greenlist_logits(\n            scores=scores, greenlist_mask=green_tokens_mask, greenlist_bias=self.delta\n        )\n        return scores\n", "path": "server/text_generation_server/utils/watermark.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright 2023 Authors of \"A Watermark for Large Language Models\"\n# available at https://arxiv.org/abs/2301.10226\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport os\n\nimport torch\nfrom transformers import LogitsProcessor\nfrom typing import List, Union\n\nGAMMA = float(os.getenv(\"WATERMARK_GAMMA\", 0.5))\nDELTA = float(os.getenv(\"WATERMARK_DELTA\", 2.0))\n\n\nclass WatermarkLogitsProcessor(LogitsProcessor):\n    def __init__(\n        self,\n        gamma: float = GAMMA,\n        delta: float = DELTA,\n        hash_key: int = 15485863,  # just a large prime number to create a rng seed with sufficient bit width\n        device: str = \"cpu\",\n    ):\n        # watermarking parameters\n        self.gamma = gamma\n        self.delta = delta\n        self.rng = torch.Generator(device=device)\n        self.hash_key = hash_key\n\n    def _seed_rng(self, input_ids: Union[List[int], torch.LongTensor]):\n        if isinstance(input_ids, list):\n            assert (\n                len(input_ids) >= 1\n            ), \"requires at least a 1 token prefix sequence to seed rng\"\n            prev_token = input_ids[-1]\n        else:\n            assert len(input_ids) == 1\n            input_ids = input_ids[0]\n            assert (\n                input_ids.shape[-1] >= 1\n            ), \"requires at least a 1 token prefix sequence to seed rng\"\n            prev_token = input_ids[-1].item()\n        self.rng.manual_seed(self.hash_key * prev_token)\n\n    def _get_greenlist_ids(\n        self,\n        input_ids: Union[List[int], torch.LongTensor],\n        max_value: int,\n        device: torch.device,\n    ) -> List[int]:\n        # seed the rng using the previous tokens/prefix\n        self._seed_rng(input_ids)\n\n        greenlist_size = int(max_value * self.gamma)\n        vocab_permutation = torch.randperm(max_value, device=device, generator=self.rng)\n        greenlist_ids = vocab_permutation[:greenlist_size]\n        return greenlist_ids\n\n    @staticmethod\n    def _calc_greenlist_mask(\n        scores: torch.FloatTensor, greenlist_token_ids\n    ) -> torch.BoolTensor:\n        green_tokens_mask = torch.zeros_like(scores)\n        green_tokens_mask[-1, greenlist_token_ids] = 1\n        final_mask = green_tokens_mask.bool()\n        return final_mask\n\n    @staticmethod\n    def _bias_greenlist_logits(\n        scores: torch.Tensor, greenlist_mask: torch.Tensor, greenlist_bias: float\n    ) -> torch.Tensor:\n        scores[greenlist_mask] = scores[greenlist_mask] + greenlist_bias\n        return scores\n\n    def __call__(\n        self, input_ids: Union[List[int], torch.LongTensor], scores: torch.FloatTensor\n    ) -> torch.FloatTensor:\n        greenlist_ids = self._get_greenlist_ids(\n            input_ids, scores.shape[-1], scores.device\n        )\n        green_tokens_mask = self._calc_greenlist_mask(\n            scores=scores, greenlist_token_ids=greenlist_ids\n        )\n\n        scores = self._bias_greenlist_logits(\n            scores=scores, greenlist_mask=green_tokens_mask, greenlist_bias=self.delta\n        )\n        return scores\n", "path": "server/text_generation_server/utils/watermark.py"}]}