We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Limit the number of fields (columns) in /croissant
https://huggingface.co/datasets/mstz/arcene has 10K columns, and thus the /croissant response is 3.7MB (120K lines when the JSON is prettified).

We already limit the number of supported columns to 1K: https://github.com/huggingface/datasets-server/blob/main/chart/values.yaml#L204

We could apply the same limit, and mention it in the description of the recordset.

From Slack (internal): https://huggingface.slack.com/archives/C04L6P8KNQ5/p1705224717632239 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `services/api/src/api/routes/croissant.py`
Content:
```
1 import logging
2 import re
3 from collections.abc import Mapping
4 from http import HTTPStatus
5 from itertools import islice
6 from typing import Any, Optional
7 
8 from datasets import ClassLabel, Features, Image, Value
9 from libapi.authentication import auth_check
10 from libapi.exceptions import (
11     ApiError,
12     MissingRequiredParameterError,
13     UnexpectedApiError,
14 )
15 from libapi.request import get_request_parameter
16 from libapi.utils import (
17     Endpoint,
18     are_valid_parameters,
19     get_cache_entry_from_steps,
20     get_json_api_error_response,
21     get_json_error_response,
22     get_json_ok_response,
23 )
24 from libcommon.constants import DATASET_INFO_KINDS
25 from libcommon.prometheus import StepProfiler
26 from libcommon.storage_client import StorageClient
27 from starlette.requests import Request
28 from starlette.responses import Response
29 
30 MAX_CONFIGS = 100
31 
32 HF_TO_CROISSANT_VALUE_TYPE = {
33     "string": "sc:Text",
34     "int32": "sc:Integer",
35     "int64": "sc:Integer",
36     "float32": "sc:Float",
37     "float64": "sc:Float",
38     "bool": "sc:Boolean",
39 }
40 
41 NAME_PATTERN_REGEX = "[^a-zA-Z0-9\\-_\\.]"
42 
43 
44 def _escape_name(name: str, names: set[str]) -> str:
45     """Escapes names in Croissant.
46 
47     Reasons:
48     - `/` are used in the syntax as delimiters. So we replace them.
49     - Two FileObject/FileSet/RecordSet/Fields cannot have the same name. So we append a postfix in case it happens.
50 
51     Args:
52         name: The initial non-escaped name.
53         names: The set of already existing names.
54     Returns:
55         The escaped name.
56     """
57     escaped_name = re.sub(NAME_PATTERN_REGEX, "_", name)
58     while escaped_name in names:
59         escaped_name = f"{escaped_name}_0"
60     names.add(escaped_name)
61     return escaped_name
62 
63 
64 def get_croissant_from_dataset_infos(dataset: str, infos: list[Mapping[str, Any]], partial: bool) -> Mapping[str, Any]:
65     repo_name = "repo"
66     names: set[str] = set(repo_name)
67     distribution = [
68         {
69             "@type": "sc:FileObject",
70             "name": repo_name,
71             "description": "The Hugging Face git repository.",
72             "contentUrl": f"https://huggingface.co/datasets/{dataset}/tree/refs%2Fconvert%2Fparquet",
73             "encodingFormat": "git+https",
74             "sha256": "https://github.com/mlcommons/croissant/issues/80",
75         }
76     ]
77     record_set = []
78     for info in infos:
79         config = info["config_name"]
80         features = Features.from_dict(info["features"])
81         fields: list[dict[str, Any]] = []
82         splits = list(info["splits"])
83         distribution_name = _escape_name(f"parquet-files-for-config-{config}", names)
84         distribution.append(
85             {
86                 "@type": "sc:FileSet",
87                 "name": distribution_name,
88                 "containedIn": repo_name,
89                 "encodingFormat": "application/x-parquet",
90                 "includes": f"{config}/*/*.parquet",
91             }
92         )
93         skipped_columns = []
94         for column, feature in features.items():
95             fields_names: set[str] = set()
96             if isinstance(feature, Value) and feature.dtype in HF_TO_CROISSANT_VALUE_TYPE:
97                 fields.append(
98                     {
99                         "@type": "ml:Field",
100                         "name": _escape_name(column, fields_names),
101                         "description": f"Column '{column}' from the Hugging Face parquet file.",
102                         "dataType": HF_TO_CROISSANT_VALUE_TYPE[feature.dtype],
103                         "source": {"distribution": distribution_name, "extract": {"column": column}},
104                     }
105                 )
106             elif isinstance(feature, Image):
107                 fields.append(
108                     {
109                         "@type": "ml:Field",
110                         "name": _escape_name(column, fields_names),
111                         "description": f"Image column '{column}' from the Hugging Face parquet file.",
112                         "dataType": "sc:ImageObject",
113                         "source": {
114                             "distribution": distribution_name,
115                             "extract": {"column": column},
116                             "transform": {"jsonPath": "bytes"},
117                         },
118                     }
119                 )
120             elif isinstance(feature, ClassLabel):
121                 fields.append(
122                     {
123                         "@type": "ml:Field",
124                         "name": _escape_name(column, fields_names),
125                         "description": f"ClassLabel column '{column}' from the Hugging Face parquet file.\nLabels:\n"
126                         + ", ".join(f"{name} ({i})" for i, name in enumerate(feature.names)),
127                         "dataType": "sc:Integer",
128                         "source": {"distribution": distribution_name, "extract": {"column": column}},
129                     }
130                 )
131             else:
132                 skipped_columns.append(column)
133         record_set_name = config if config != dataset else f"record_set_{config}"
134         description = f"{dataset} - '{config}' subset"
135         if partial:
136             description += " (first 5GB)"
137         description_body = ""
138         if len(splits) > 1:
139             description_body += f"\n- {len(splits)} split{'s' if len(splits) > 1 else ''}: {', '.join(splits)}"
140         if skipped_columns:
141             description_body += f"\n- {len(skipped_columns)} skipped column{'s' if len(skipped_columns) > 1 else ''}: {', '.join(skipped_columns)}"
142         if description_body:
143             description += "\n\nAdditional information:"
144             description += description_body
145         record_set.append(
146             {
147                 "@type": "ml:RecordSet",
148                 "name": _escape_name(record_set_name, names),
149                 "description": description,
150                 "field": fields,
151             }
152         )
153     return {
154         "@context": {
155             "@language": "en",
156             "@vocab": "https://schema.org/",
157             "column": "ml:column",
158             "data": {
159                 "@id": "ml:data",
160                 "@type": "@json",
161             },
162             "dataType": {
163                 "@id": "ml:dataType",
164                 "@type": "@vocab",
165             },
166             "extract": "ml:extract",
167             "field": "ml:field",
168             "fileProperty": "ml:fileProperty",
169             "format": "ml:format",
170             "includes": "ml:includes",
171             "isEnumeration": "ml:isEnumeration",
172             "jsonPath": "ml:jsonPath",
173             "ml": "http://mlcommons.org/schema/",
174             "parentField": "ml:parentField",
175             "path": "ml:path",
176             "recordSet": "ml:recordSet",
177             "references": "ml:references",
178             "regex": "ml:regex",
179             "repeated": "ml:repeated",
180             "replace": "ml:replace",
181             "sc": "https://schema.org/",
182             "separator": "ml:separator",
183             "source": "ml:source",
184             "subField": "ml:subField",
185             "transform": "ml:transform",
186         },
187         "@type": "sc:Dataset",
188         "name": _escape_name(dataset, names),
189         "description": f"{dataset} dataset hosted on Hugging Face and contributed by the HF Datasets community",
190         "url": f"https://huggingface.co/datasets/{dataset}",
191         "distribution": distribution,
192         "recordSet": record_set,
193     }
194 
195 
196 def create_croissant_endpoint(
197     hf_endpoint: str,
198     blocked_datasets: list[str],
199     hf_token: Optional[str] = None,
200     hf_jwt_public_keys: Optional[list[str]] = None,
201     hf_jwt_algorithm: Optional[str] = None,
202     external_auth_url: Optional[str] = None,
203     hf_timeout_seconds: Optional[float] = None,
204     max_age_long: int = 0,
205     max_age_short: int = 0,
206     storage_clients: Optional[list[StorageClient]] = None,
207 ) -> Endpoint:
208     async def croissant_endpoint(request: Request) -> Response:
209         endpoint_name = "croissant"
210         context = f"endpoint: {endpoint_name}"
211         revision: Optional[str] = None
212         with StepProfiler(method="croissant_endpoint", step="all", context=context):
213             try:
214                 with StepProfiler(
215                     method="croissant_endpoint",
216                     step="validate parameters and get processing steps",
217                     context=context,
218                 ):
219                     dataset = get_request_parameter(request, "dataset")
220                     logging.debug(f"endpoint={endpoint_name} dataset={dataset}")
221                     if not are_valid_parameters([dataset]):
222                         raise MissingRequiredParameterError("Parameter 'dataset' is required")
223                 # if auth_check fails, it will raise an exception that will be caught below
224                 with StepProfiler(method="croissant_endpoint", step="check authentication", context=context):
225                     await auth_check(
226                         dataset,
227                         external_auth_url=external_auth_url,
228                         request=request,
229                         hf_jwt_public_keys=hf_jwt_public_keys,
230                         hf_jwt_algorithm=hf_jwt_algorithm,
231                         hf_timeout_seconds=hf_timeout_seconds,
232                     )
233                 # getting result based on processing steps
234                 with StepProfiler(method="croissant_endpoint", step="get info cache entry", context=context):
235                     info_result = get_cache_entry_from_steps(
236                         processing_step_names=DATASET_INFO_KINDS,
237                         dataset=dataset,
238                         config=None,
239                         split=None,
240                         hf_endpoint=hf_endpoint,
241                         hf_token=hf_token,
242                         blocked_datasets=blocked_datasets,
243                         hf_timeout_seconds=hf_timeout_seconds,
244                         storage_clients=storage_clients,
245                     )
246                 content = info_result["content"]
247                 http_status = info_result["http_status"]
248                 error_code = info_result["error_code"]
249                 revision = info_result["dataset_git_revision"]
250                 if http_status == HTTPStatus.OK:
251                     infos = list(islice(content["dataset_info"].values(), MAX_CONFIGS))
252                     partial = content["partial"]
253                     with StepProfiler(method="croissant_endpoint", step="generate croissant json", context=context):
254                         croissant = get_croissant_from_dataset_infos(dataset=dataset, infos=infos, partial=partial)
255                     with StepProfiler(method="croissant_endpoint", step="generate OK response", context=context):
256                         return get_json_ok_response(content=croissant, max_age=max_age_long, revision=revision)
257                 else:
258                     with StepProfiler(method="croissant_endpoint", step="generate error response", context=context):
259                         return get_json_error_response(
260                             content=content,
261                             status_code=http_status,
262                             max_age=max_age_short,
263                             error_code=error_code,
264                             revision=revision,
265                         )
266             except Exception as e:
267                 error = e if isinstance(e, ApiError) else UnexpectedApiError("Unexpected error.", e)
268                 with StepProfiler(method="croissant_endpoint", step="generate API error response", context=context):
269                     return get_json_api_error_response(error=error, max_age=max_age_short, revision=revision)
270 
271     return croissant_endpoint
272
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/services/api/src/api/routes/croissant.py b/services/api/src/api/routes/croissant.py
--- a/services/api/src/api/routes/croissant.py
+++ b/services/api/src/api/routes/croissant.py
@@ -28,6 +28,9 @@
 from starlette.responses import Response
 
 MAX_CONFIGS = 100
+MAX_COLUMNS = 1_000
+# ^ same value as the default for FIRST_ROWS_COLUMNS_MAX_NUMBER (see services/worker)
+
 
 HF_TO_CROISSANT_VALUE_TYPE = {
     "string": "sc:Text",
@@ -76,6 +79,7 @@
     ]
     record_set = []
     for info in infos:
+        description_body = ""
         config = info["config_name"]
         features = Features.from_dict(info["features"])
         fields: list[dict[str, Any]] = []
@@ -92,6 +96,9 @@
         )
         skipped_columns = []
         for column, feature in features.items():
+            if len(fields) >= MAX_COLUMNS:
+                description_body += f"\n- {len(features) - MAX_COLUMNS} skipped column{'s' if len(features) - MAX_COLUMNS > 1 else ''} (max number of columns reached)"
+                break
             fields_names: set[str] = set()
             if isinstance(feature, Value) and feature.dtype in HF_TO_CROISSANT_VALUE_TYPE:
                 fields.append(
@@ -134,7 +141,6 @@
         description = f"{dataset} - '{config}' subset"
         if partial:
             description += " (first 5GB)"
-        description_body = ""
         if len(splits) > 1:
             description_body += f"\n- {len(splits)} split{'s' if len(splits) > 1 else ''}: {', '.join(splits)}"
         if skipped_columns:

{"golden_diff": "diff --git a/services/api/src/api/routes/croissant.py b/services/api/src/api/routes/croissant.py\n--- a/services/api/src/api/routes/croissant.py\n+++ b/services/api/src/api/routes/croissant.py\n@@ -28,6 +28,9 @@\n from starlette.responses import Response\n \n MAX_CONFIGS = 100\n+MAX_COLUMNS = 1_000\n+# ^ same value as the default for FIRST_ROWS_COLUMNS_MAX_NUMBER (see services/worker)\n+\n \n HF_TO_CROISSANT_VALUE_TYPE = {\n     \"string\": \"sc:Text\",\n@@ -76,6 +79,7 @@\n     ]\n     record_set = []\n     for info in infos:\n+        description_body = \"\"\n         config = info[\"config_name\"]\n         features = Features.from_dict(info[\"features\"])\n         fields: list[dict[str, Any]] = []\n@@ -92,6 +96,9 @@\n         )\n         skipped_columns = []\n         for column, feature in features.items():\n+            if len(fields) >= MAX_COLUMNS:\n+                description_body += f\"\\n- {len(features) - MAX_COLUMNS} skipped column{'s' if len(features) - MAX_COLUMNS > 1 else ''} (max number of columns reached)\"\n+                break\n             fields_names: set[str] = set()\n             if isinstance(feature, Value) and feature.dtype in HF_TO_CROISSANT_VALUE_TYPE:\n                 fields.append(\n@@ -134,7 +141,6 @@\n         description = f\"{dataset} - '{config}' subset\"\n         if partial:\n             description += \" (first 5GB)\"\n-        description_body = \"\"\n         if len(splits) > 1:\n             description_body += f\"\\n- {len(splits)} split{'s' if len(splits) > 1 else ''}: {', '.join(splits)}\"\n         if skipped_columns:\n", "issue": "Limit the number of fields (columns) in /croissant\nhttps://huggingface.co/datasets/mstz/arcene has 10K columns, and thus the /croissant response is 3.7MB (120K lines when the JSON is prettified).\r\n\r\nWe already limit the number of supported columns to 1K: https://github.com/huggingface/datasets-server/blob/main/chart/values.yaml#L204\r\n\r\nWe could apply the same limit, and mention it in the description of the recordset.\r\n\r\nFrom Slack (internal): https://huggingface.slack.com/archives/C04L6P8KNQ5/p1705224717632239 \n", "before_files": [{"content": "import logging\nimport re\nfrom collections.abc import Mapping\nfrom http import HTTPStatus\nfrom itertools import islice\nfrom typing import Any, Optional\n\nfrom datasets import ClassLabel, Features, Image, Value\nfrom libapi.authentication import auth_check\nfrom libapi.exceptions import (\n    ApiError,\n    MissingRequiredParameterError,\n    UnexpectedApiError,\n)\nfrom libapi.request import get_request_parameter\nfrom libapi.utils import (\n    Endpoint,\n    are_valid_parameters,\n    get_cache_entry_from_steps,\n    get_json_api_error_response,\n    get_json_error_response,\n    get_json_ok_response,\n)\nfrom libcommon.constants import DATASET_INFO_KINDS\nfrom libcommon.prometheus import StepProfiler\nfrom libcommon.storage_client import StorageClient\nfrom starlette.requests import Request\nfrom starlette.responses import Response\n\nMAX_CONFIGS = 100\n\nHF_TO_CROISSANT_VALUE_TYPE = {\n    \"string\": \"sc:Text\",\n    \"int32\": \"sc:Integer\",\n    \"int64\": \"sc:Integer\",\n    \"float32\": \"sc:Float\",\n    \"float64\": \"sc:Float\",\n    \"bool\": \"sc:Boolean\",\n}\n\nNAME_PATTERN_REGEX = \"[^a-zA-Z0-9\\\\-_\\\\.]\"\n\n\ndef _escape_name(name: str, names: set[str]) -> str:\n    \"\"\"Escapes names in Croissant.\n\n    Reasons:\n    - `/` are used in the syntax as delimiters. So we replace them.\n    - Two FileObject/FileSet/RecordSet/Fields cannot have the same name. So we append a postfix in case it happens.\n\n    Args:\n        name: The initial non-escaped name.\n        names: The set of already existing names.\n    Returns:\n        The escaped name.\n    \"\"\"\n    escaped_name = re.sub(NAME_PATTERN_REGEX, \"_\", name)\n    while escaped_name in names:\n        escaped_name = f\"{escaped_name}_0\"\n    names.add(escaped_name)\n    return escaped_name\n\n\ndef get_croissant_from_dataset_infos(dataset: str, infos: list[Mapping[str, Any]], partial: bool) -> Mapping[str, Any]:\n    repo_name = \"repo\"\n    names: set[str] = set(repo_name)\n    distribution = [\n        {\n            \"@type\": \"sc:FileObject\",\n            \"name\": repo_name,\n            \"description\": \"The Hugging Face git repository.\",\n            \"contentUrl\": f\"https://huggingface.co/datasets/{dataset}/tree/refs%2Fconvert%2Fparquet\",\n            \"encodingFormat\": \"git+https\",\n            \"sha256\": \"https://github.com/mlcommons/croissant/issues/80\",\n        }\n    ]\n    record_set = []\n    for info in infos:\n        config = info[\"config_name\"]\n        features = Features.from_dict(info[\"features\"])\n        fields: list[dict[str, Any]] = []\n        splits = list(info[\"splits\"])\n        distribution_name = _escape_name(f\"parquet-files-for-config-{config}\", names)\n        distribution.append(\n            {\n                \"@type\": \"sc:FileSet\",\n                \"name\": distribution_name,\n                \"containedIn\": repo_name,\n                \"encodingFormat\": \"application/x-parquet\",\n                \"includes\": f\"{config}/*/*.parquet\",\n            }\n        )\n        skipped_columns = []\n        for column, feature in features.items():\n            fields_names: set[str] = set()\n            if isinstance(feature, Value) and feature.dtype in HF_TO_CROISSANT_VALUE_TYPE:\n                fields.append(\n                    {\n                        \"@type\": \"ml:Field\",\n                        \"name\": _escape_name(column, fields_names),\n                        \"description\": f\"Column '{column}' from the Hugging Face parquet file.\",\n                        \"dataType\": HF_TO_CROISSANT_VALUE_TYPE[feature.dtype],\n                        \"source\": {\"distribution\": distribution_name, \"extract\": {\"column\": column}},\n                    }\n                )\n            elif isinstance(feature, Image):\n                fields.append(\n                    {\n                        \"@type\": \"ml:Field\",\n                        \"name\": _escape_name(column, fields_names),\n                        \"description\": f\"Image column '{column}' from the Hugging Face parquet file.\",\n                        \"dataType\": \"sc:ImageObject\",\n                        \"source\": {\n                            \"distribution\": distribution_name,\n                            \"extract\": {\"column\": column},\n                            \"transform\": {\"jsonPath\": \"bytes\"},\n                        },\n                    }\n                )\n            elif isinstance(feature, ClassLabel):\n                fields.append(\n                    {\n                        \"@type\": \"ml:Field\",\n                        \"name\": _escape_name(column, fields_names),\n                        \"description\": f\"ClassLabel column '{column}' from the Hugging Face parquet file.\\nLabels:\\n\"\n                        + \", \".join(f\"{name} ({i})\" for i, name in enumerate(feature.names)),\n                        \"dataType\": \"sc:Integer\",\n                        \"source\": {\"distribution\": distribution_name, \"extract\": {\"column\": column}},\n                    }\n                )\n            else:\n                skipped_columns.append(column)\n        record_set_name = config if config != dataset else f\"record_set_{config}\"\n        description = f\"{dataset} - '{config}' subset\"\n        if partial:\n            description += \" (first 5GB)\"\n        description_body = \"\"\n        if len(splits) > 1:\n            description_body += f\"\\n- {len(splits)} split{'s' if len(splits) > 1 else ''}: {', '.join(splits)}\"\n        if skipped_columns:\n            description_body += f\"\\n- {len(skipped_columns)} skipped column{'s' if len(skipped_columns) > 1 else ''}: {', '.join(skipped_columns)}\"\n        if description_body:\n            description += \"\\n\\nAdditional information:\"\n            description += description_body\n        record_set.append(\n            {\n                \"@type\": \"ml:RecordSet\",\n                \"name\": _escape_name(record_set_name, names),\n                \"description\": description,\n                \"field\": fields,\n            }\n        )\n    return {\n        \"@context\": {\n            \"@language\": \"en\",\n            \"@vocab\": \"https://schema.org/\",\n            \"column\": \"ml:column\",\n            \"data\": {\n                \"@id\": \"ml:data\",\n                \"@type\": \"@json\",\n            },\n            \"dataType\": {\n                \"@id\": \"ml:dataType\",\n                \"@type\": \"@vocab\",\n            },\n            \"extract\": \"ml:extract\",\n            \"field\": \"ml:field\",\n            \"fileProperty\": \"ml:fileProperty\",\n            \"format\": \"ml:format\",\n            \"includes\": \"ml:includes\",\n            \"isEnumeration\": \"ml:isEnumeration\",\n            \"jsonPath\": \"ml:jsonPath\",\n            \"ml\": \"http://mlcommons.org/schema/\",\n            \"parentField\": \"ml:parentField\",\n            \"path\": \"ml:path\",\n            \"recordSet\": \"ml:recordSet\",\n            \"references\": \"ml:references\",\n            \"regex\": \"ml:regex\",\n            \"repeated\": \"ml:repeated\",\n            \"replace\": \"ml:replace\",\n            \"sc\": \"https://schema.org/\",\n            \"separator\": \"ml:separator\",\n            \"source\": \"ml:source\",\n            \"subField\": \"ml:subField\",\n            \"transform\": \"ml:transform\",\n        },\n        \"@type\": \"sc:Dataset\",\n        \"name\": _escape_name(dataset, names),\n        \"description\": f\"{dataset} dataset hosted on Hugging Face and contributed by the HF Datasets community\",\n        \"url\": f\"https://huggingface.co/datasets/{dataset}\",\n        \"distribution\": distribution,\n        \"recordSet\": record_set,\n    }\n\n\ndef create_croissant_endpoint(\n    hf_endpoint: str,\n    blocked_datasets: list[str],\n    hf_token: Optional[str] = None,\n    hf_jwt_public_keys: Optional[list[str]] = None,\n    hf_jwt_algorithm: Optional[str] = None,\n    external_auth_url: Optional[str] = None,\n    hf_timeout_seconds: Optional[float] = None,\n    max_age_long: int = 0,\n    max_age_short: int = 0,\n    storage_clients: Optional[list[StorageClient]] = None,\n) -> Endpoint:\n    async def croissant_endpoint(request: Request) -> Response:\n        endpoint_name = \"croissant\"\n        context = f\"endpoint: {endpoint_name}\"\n        revision: Optional[str] = None\n        with StepProfiler(method=\"croissant_endpoint\", step=\"all\", context=context):\n            try:\n                with StepProfiler(\n                    method=\"croissant_endpoint\",\n                    step=\"validate parameters and get processing steps\",\n                    context=context,\n                ):\n                    dataset = get_request_parameter(request, \"dataset\")\n                    logging.debug(f\"endpoint={endpoint_name} dataset={dataset}\")\n                    if not are_valid_parameters([dataset]):\n                        raise MissingRequiredParameterError(\"Parameter 'dataset' is required\")\n                # if auth_check fails, it will raise an exception that will be caught below\n                with StepProfiler(method=\"croissant_endpoint\", step=\"check authentication\", context=context):\n                    await auth_check(\n                        dataset,\n                        external_auth_url=external_auth_url,\n                        request=request,\n                        hf_jwt_public_keys=hf_jwt_public_keys,\n                        hf_jwt_algorithm=hf_jwt_algorithm,\n                        hf_timeout_seconds=hf_timeout_seconds,\n                    )\n                # getting result based on processing steps\n                with StepProfiler(method=\"croissant_endpoint\", step=\"get info cache entry\", context=context):\n                    info_result = get_cache_entry_from_steps(\n                        processing_step_names=DATASET_INFO_KINDS,\n                        dataset=dataset,\n                        config=None,\n                        split=None,\n                        hf_endpoint=hf_endpoint,\n                        hf_token=hf_token,\n                        blocked_datasets=blocked_datasets,\n                        hf_timeout_seconds=hf_timeout_seconds,\n                        storage_clients=storage_clients,\n                    )\n                content = info_result[\"content\"]\n                http_status = info_result[\"http_status\"]\n                error_code = info_result[\"error_code\"]\n                revision = info_result[\"dataset_git_revision\"]\n                if http_status == HTTPStatus.OK:\n                    infos = list(islice(content[\"dataset_info\"].values(), MAX_CONFIGS))\n                    partial = content[\"partial\"]\n                    with StepProfiler(method=\"croissant_endpoint\", step=\"generate croissant json\", context=context):\n                        croissant = get_croissant_from_dataset_infos(dataset=dataset, infos=infos, partial=partial)\n                    with StepProfiler(method=\"croissant_endpoint\", step=\"generate OK response\", context=context):\n                        return get_json_ok_response(content=croissant, max_age=max_age_long, revision=revision)\n                else:\n                    with StepProfiler(method=\"croissant_endpoint\", step=\"generate error response\", context=context):\n                        return get_json_error_response(\n                            content=content,\n                            status_code=http_status,\n                            max_age=max_age_short,\n                            error_code=error_code,\n                            revision=revision,\n                        )\n            except Exception as e:\n                error = e if isinstance(e, ApiError) else UnexpectedApiError(\"Unexpected error.\", e)\n                with StepProfiler(method=\"croissant_endpoint\", step=\"generate API error response\", context=context):\n                    return get_json_api_error_response(error=error, max_age=max_age_short, revision=revision)\n\n    return croissant_endpoint\n", "path": "services/api/src/api/routes/croissant.py"}], "after_files": [{"content": "import logging\nimport re\nfrom collections.abc import Mapping\nfrom http import HTTPStatus\nfrom itertools import islice\nfrom typing import Any, Optional\n\nfrom datasets import ClassLabel, Features, Image, Value\nfrom libapi.authentication import auth_check\nfrom libapi.exceptions import (\n    ApiError,\n    MissingRequiredParameterError,\n    UnexpectedApiError,\n)\nfrom libapi.request import get_request_parameter\nfrom libapi.utils import (\n    Endpoint,\n    are_valid_parameters,\n    get_cache_entry_from_steps,\n    get_json_api_error_response,\n    get_json_error_response,\n    get_json_ok_response,\n)\nfrom libcommon.constants import DATASET_INFO_KINDS\nfrom libcommon.prometheus import StepProfiler\nfrom libcommon.storage_client import StorageClient\nfrom starlette.requests import Request\nfrom starlette.responses import Response\n\nMAX_CONFIGS = 100\nMAX_COLUMNS = 1_000\n# ^ same value as the default for FIRST_ROWS_COLUMNS_MAX_NUMBER (see services/worker)\n\n\nHF_TO_CROISSANT_VALUE_TYPE = {\n    \"string\": \"sc:Text\",\n    \"int32\": \"sc:Integer\",\n    \"int64\": \"sc:Integer\",\n    \"float32\": \"sc:Float\",\n    \"float64\": \"sc:Float\",\n    \"bool\": \"sc:Boolean\",\n}\n\nNAME_PATTERN_REGEX = \"[^a-zA-Z0-9\\\\-_\\\\.]\"\n\n\ndef _escape_name(name: str, names: set[str]) -> str:\n    \"\"\"Escapes names in Croissant.\n\n    Reasons:\n    - `/` are used in the syntax as delimiters. So we replace them.\n    - Two FileObject/FileSet/RecordSet/Fields cannot have the same name. So we append a postfix in case it happens.\n\n    Args:\n        name: The initial non-escaped name.\n        names: The set of already existing names.\n    Returns:\n        The escaped name.\n    \"\"\"\n    escaped_name = re.sub(NAME_PATTERN_REGEX, \"_\", name)\n    while escaped_name in names:\n        escaped_name = f\"{escaped_name}_0\"\n    names.add(escaped_name)\n    return escaped_name\n\n\ndef get_croissant_from_dataset_infos(dataset: str, infos: list[Mapping[str, Any]], partial: bool) -> Mapping[str, Any]:\n    repo_name = \"repo\"\n    names: set[str] = set(repo_name)\n    distribution = [\n        {\n            \"@type\": \"sc:FileObject\",\n            \"name\": repo_name,\n            \"description\": \"The Hugging Face git repository.\",\n            \"contentUrl\": f\"https://huggingface.co/datasets/{dataset}/tree/refs%2Fconvert%2Fparquet\",\n            \"encodingFormat\": \"git+https\",\n            \"sha256\": \"https://github.com/mlcommons/croissant/issues/80\",\n        }\n    ]\n    record_set = []\n    for info in infos:\n        description_body = \"\"\n        config = info[\"config_name\"]\n        features = Features.from_dict(info[\"features\"])\n        fields: list[dict[str, Any]] = []\n        splits = list(info[\"splits\"])\n        distribution_name = _escape_name(f\"parquet-files-for-config-{config}\", names)\n        distribution.append(\n            {\n                \"@type\": \"sc:FileSet\",\n                \"name\": distribution_name,\n                \"containedIn\": repo_name,\n                \"encodingFormat\": \"application/x-parquet\",\n                \"includes\": f\"{config}/*/*.parquet\",\n            }\n        )\n        skipped_columns = []\n        for column, feature in features.items():\n            if len(fields) >= MAX_COLUMNS:\n                description_body += f\"\\n- {len(features) - MAX_COLUMNS} skipped column{'s' if len(features) - MAX_COLUMNS > 1 else ''} (max number of columns reached)\"\n                break\n            fields_names: set[str] = set()\n            if isinstance(feature, Value) and feature.dtype in HF_TO_CROISSANT_VALUE_TYPE:\n                fields.append(\n                    {\n                        \"@type\": \"ml:Field\",\n                        \"name\": _escape_name(column, fields_names),\n                        \"description\": f\"Column '{column}' from the Hugging Face parquet file.\",\n                        \"dataType\": HF_TO_CROISSANT_VALUE_TYPE[feature.dtype],\n                        \"source\": {\"distribution\": distribution_name, \"extract\": {\"column\": column}},\n                    }\n                )\n            elif isinstance(feature, Image):\n                fields.append(\n                    {\n                        \"@type\": \"ml:Field\",\n                        \"name\": _escape_name(column, fields_names),\n                        \"description\": f\"Image column '{column}' from the Hugging Face parquet file.\",\n                        \"dataType\": \"sc:ImageObject\",\n                        \"source\": {\n                            \"distribution\": distribution_name,\n                            \"extract\": {\"column\": column},\n                            \"transform\": {\"jsonPath\": \"bytes\"},\n                        },\n                    }\n                )\n            elif isinstance(feature, ClassLabel):\n                fields.append(\n                    {\n                        \"@type\": \"ml:Field\",\n                        \"name\": _escape_name(column, fields_names),\n                        \"description\": f\"ClassLabel column '{column}' from the Hugging Face parquet file.\\nLabels:\\n\"\n                        + \", \".join(f\"{name} ({i})\" for i, name in enumerate(feature.names)),\n                        \"dataType\": \"sc:Integer\",\n                        \"source\": {\"distribution\": distribution_name, \"extract\": {\"column\": column}},\n                    }\n                )\n            else:\n                skipped_columns.append(column)\n        record_set_name = config if config != dataset else f\"record_set_{config}\"\n        description = f\"{dataset} - '{config}' subset\"\n        if partial:\n            description += \" (first 5GB)\"\n        if len(splits) > 1:\n            description_body += f\"\\n- {len(splits)} split{'s' if len(splits) > 1 else ''}: {', '.join(splits)}\"\n        if skipped_columns:\n            description_body += f\"\\n- {len(skipped_columns)} skipped column{'s' if len(skipped_columns) > 1 else ''}: {', '.join(skipped_columns)}\"\n        if description_body:\n            description += \"\\n\\nAdditional information:\"\n            description += description_body\n        record_set.append(\n            {\n                \"@type\": \"ml:RecordSet\",\n                \"name\": _escape_name(record_set_name, names),\n                \"description\": description,\n                \"field\": fields,\n            }\n        )\n    return {\n        \"@context\": {\n            \"@language\": \"en\",\n            \"@vocab\": \"https://schema.org/\",\n            \"column\": \"ml:column\",\n            \"data\": {\n                \"@id\": \"ml:data\",\n                \"@type\": \"@json\",\n            },\n            \"dataType\": {\n                \"@id\": \"ml:dataType\",\n                \"@type\": \"@vocab\",\n            },\n            \"extract\": \"ml:extract\",\n            \"field\": \"ml:field\",\n            \"fileProperty\": \"ml:fileProperty\",\n            \"format\": \"ml:format\",\n            \"includes\": \"ml:includes\",\n            \"isEnumeration\": \"ml:isEnumeration\",\n            \"jsonPath\": \"ml:jsonPath\",\n            \"ml\": \"http://mlcommons.org/schema/\",\n            \"parentField\": \"ml:parentField\",\n            \"path\": \"ml:path\",\n            \"recordSet\": \"ml:recordSet\",\n            \"references\": \"ml:references\",\n            \"regex\": \"ml:regex\",\n            \"repeated\": \"ml:repeated\",\n            \"replace\": \"ml:replace\",\n            \"sc\": \"https://schema.org/\",\n            \"separator\": \"ml:separator\",\n            \"source\": \"ml:source\",\n            \"subField\": \"ml:subField\",\n            \"transform\": \"ml:transform\",\n        },\n        \"@type\": \"sc:Dataset\",\n        \"name\": _escape_name(dataset, names),\n        \"description\": f\"{dataset} dataset hosted on Hugging Face and contributed by the HF Datasets community\",\n        \"url\": f\"https://huggingface.co/datasets/{dataset}\",\n        \"distribution\": distribution,\n        \"recordSet\": record_set,\n    }\n\n\ndef create_croissant_endpoint(\n    hf_endpoint: str,\n    blocked_datasets: list[str],\n    hf_token: Optional[str] = None,\n    hf_jwt_public_keys: Optional[list[str]] = None,\n    hf_jwt_algorithm: Optional[str] = None,\n    external_auth_url: Optional[str] = None,\n    hf_timeout_seconds: Optional[float] = None,\n    max_age_long: int = 0,\n    max_age_short: int = 0,\n    storage_clients: Optional[list[StorageClient]] = None,\n) -> Endpoint:\n    async def croissant_endpoint(request: Request) -> Response:\n        endpoint_name = \"croissant\"\n        context = f\"endpoint: {endpoint_name}\"\n        revision: Optional[str] = None\n        with StepProfiler(method=\"croissant_endpoint\", step=\"all\", context=context):\n            try:\n                with StepProfiler(\n                    method=\"croissant_endpoint\",\n                    step=\"validate parameters and get processing steps\",\n                    context=context,\n                ):\n                    dataset = get_request_parameter(request, \"dataset\")\n                    logging.debug(f\"endpoint={endpoint_name} dataset={dataset}\")\n                    if not are_valid_parameters([dataset]):\n                        raise MissingRequiredParameterError(\"Parameter 'dataset' is required\")\n                # if auth_check fails, it will raise an exception that will be caught below\n                with StepProfiler(method=\"croissant_endpoint\", step=\"check authentication\", context=context):\n                    await auth_check(\n                        dataset,\n                        external_auth_url=external_auth_url,\n                        request=request,\n                        hf_jwt_public_keys=hf_jwt_public_keys,\n                        hf_jwt_algorithm=hf_jwt_algorithm,\n                        hf_timeout_seconds=hf_timeout_seconds,\n                    )\n                # getting result based on processing steps\n                with StepProfiler(method=\"croissant_endpoint\", step=\"get info cache entry\", context=context):\n                    info_result = get_cache_entry_from_steps(\n                        processing_step_names=DATASET_INFO_KINDS,\n                        dataset=dataset,\n                        config=None,\n                        split=None,\n                        hf_endpoint=hf_endpoint,\n                        hf_token=hf_token,\n                        blocked_datasets=blocked_datasets,\n                        hf_timeout_seconds=hf_timeout_seconds,\n                        storage_clients=storage_clients,\n                    )\n                content = info_result[\"content\"]\n                http_status = info_result[\"http_status\"]\n                error_code = info_result[\"error_code\"]\n                revision = info_result[\"dataset_git_revision\"]\n                if http_status == HTTPStatus.OK:\n                    infos = list(islice(content[\"dataset_info\"].values(), MAX_CONFIGS))\n                    partial = content[\"partial\"]\n                    with StepProfiler(method=\"croissant_endpoint\", step=\"generate croissant json\", context=context):\n                        croissant = get_croissant_from_dataset_infos(dataset=dataset, infos=infos, partial=partial)\n                    with StepProfiler(method=\"croissant_endpoint\", step=\"generate OK response\", context=context):\n                        return get_json_ok_response(content=croissant, max_age=max_age_long, revision=revision)\n                else:\n                    with StepProfiler(method=\"croissant_endpoint\", step=\"generate error response\", context=context):\n                        return get_json_error_response(\n                            content=content,\n                            status_code=http_status,\n                            max_age=max_age_short,\n                            error_code=error_code,\n                            revision=revision,\n                        )\n            except Exception as e:\n                error = e if isinstance(e, ApiError) else UnexpectedApiError(\"Unexpected error.\", e)\n                with StepProfiler(method=\"croissant_endpoint\", step=\"generate API error response\", context=context):\n                    return get_json_api_error_response(error=error, max_age=max_age_short, revision=revision)\n\n    return croissant_endpoint\n", "path": "services/api/src/api/routes/croissant.py"}]}