We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Validate all headers/fields when bulk import objects
### NetBox version

v3.4.3

### Python version

3.8

### Steps to Reproduce

1. Open Prefix Bulk Import
2. Specify all required field options and add some dummy data
`prefix,status,testa,testb`
`10.10.10.0/24,active,a,b`
4. Press Import
5. Import is done without any complains


### Expected Behavior

Fields which are not stated in Field Options should throw an error, or a warning that some data was ignored.


### Observed Behavior

Import is done without any complains, Looks like all data was imported correct, although some fields were ignored

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `netbox/utilities/forms/forms.py`
Content:
```
1 import re
2 
3 from django import forms
4 from django.utils.translation import gettext as _
5 from .mixins import BootstrapMixin
6 
7 __all__ = (
8     'BulkEditForm',
9     'BulkRenameForm',
10     'ConfirmationForm',
11     'CSVModelForm',
12     'FilterForm',
13     'TableConfigForm',
14 )
15 
16 
17 class ConfirmationForm(BootstrapMixin, forms.Form):
18     """
19     A generic confirmation form. The form is not valid unless the `confirm` field is checked.
20     """
21     return_url = forms.CharField(
22         required=False,
23         widget=forms.HiddenInput()
24     )
25     confirm = forms.BooleanField(
26         required=True,
27         widget=forms.HiddenInput(),
28         initial=True
29     )
30 
31 
32 class BulkEditForm(BootstrapMixin, forms.Form):
33     """
34     Provides bulk edit support for objects.
35     """
36     nullable_fields = ()
37 
38 
39 class BulkRenameForm(BootstrapMixin, forms.Form):
40     """
41     An extendable form to be used for renaming objects in bulk.
42     """
43     find = forms.CharField(
44         strip=False
45     )
46     replace = forms.CharField(
47         strip=False,
48         required=False
49     )
50     use_regex = forms.BooleanField(
51         required=False,
52         initial=True,
53         label=_('Use regular expressions')
54     )
55 
56     def clean(self):
57         super().clean()
58 
59         # Validate regular expression in "find" field
60         if self.cleaned_data['use_regex']:
61             try:
62                 re.compile(self.cleaned_data['find'])
63             except re.error:
64                 raise forms.ValidationError({
65                     'find': "Invalid regular expression"
66                 })
67 
68 
69 class CSVModelForm(forms.ModelForm):
70     """
71     ModelForm used for the import of objects in CSV format.
72     """
73     def __init__(self, *args, headers=None, fields=None, **kwargs):
74         headers = headers or {}
75         fields = fields or []
76         super().__init__(*args, **kwargs)
77 
78         # Modify the model form to accommodate any customized to_field_name properties
79         for field, to_field in headers.items():
80             if to_field is not None:
81                 self.fields[field].to_field_name = to_field
82 
83         # Omit any fields not specified (e.g. because the form is being used to
84         # updated rather than create objects)
85         if fields:
86             for field in list(self.fields.keys()):
87                 if field not in fields:
88                     del self.fields[field]
89 
90 
91 class FilterForm(BootstrapMixin, forms.Form):
92     """
93     Base Form class for FilterSet forms.
94     """
95     q = forms.CharField(
96         required=False,
97         label=_('Search')
98     )
99 
100 
101 class TableConfigForm(BootstrapMixin, forms.Form):
102     """
103     Form for configuring user's table preferences.
104     """
105     available_columns = forms.MultipleChoiceField(
106         choices=[],
107         required=False,
108         widget=forms.SelectMultiple(
109             attrs={'size': 10, 'class': 'form-select'}
110         ),
111         label=_('Available Columns')
112     )
113     columns = forms.MultipleChoiceField(
114         choices=[],
115         required=False,
116         widget=forms.SelectMultiple(
117             attrs={'size': 10, 'class': 'form-select'}
118         ),
119         label=_('Selected Columns')
120     )
121 
122     def __init__(self, table, *args, **kwargs):
123         self.table = table
124 
125         super().__init__(*args, **kwargs)
126 
127         # Initialize columns field based on table attributes
128         self.fields['available_columns'].choices = table.available_columns
129         self.fields['columns'].choices = table.selected_columns
130 
131     @property
132     def table_name(self):
133         return self.table.__class__.__name__
134
```

Path: `netbox/utilities/forms/bulk_import.py`
Content:
```
1 import csv
2 import json
3 from io import StringIO
4 
5 import yaml
6 from django import forms
7 from django.utils.translation import gettext as _
8 
9 from core.forms.mixins import SyncedDataMixin
10 from utilities.choices import CSVDelimiterChoices, ImportFormatChoices, ImportMethodChoices
11 from utilities.constants import CSV_DELIMITERS
12 from utilities.forms.utils import parse_csv
13 from .mixins import BootstrapMixin
14 
15 
16 class BulkImportForm(BootstrapMixin, SyncedDataMixin, forms.Form):
17     import_method = forms.ChoiceField(
18         choices=ImportMethodChoices,
19         required=False
20     )
21     data = forms.CharField(
22         required=False,
23         widget=forms.Textarea(attrs={'class': 'font-monospace'}),
24         help_text=_("Enter object data in CSV, JSON or YAML format.")
25     )
26     upload_file = forms.FileField(
27         label=_("Data file"),
28         required=False
29     )
30     format = forms.ChoiceField(
31         choices=ImportFormatChoices,
32         initial=ImportFormatChoices.AUTO
33     )
34     csv_delimiter = forms.ChoiceField(
35         choices=CSVDelimiterChoices,
36         initial=CSVDelimiterChoices.AUTO,
37         label=_("CSV delimiter"),
38         help_text=_("The character which delimits CSV fields. Applies only to CSV format."),
39         required=False
40     )
41 
42     data_field = 'data'
43 
44     def clean(self):
45         super().clean()
46 
47         # Determine import method
48         import_method = self.cleaned_data.get('import_method') or ImportMethodChoices.DIRECT
49 
50         # Determine whether we're reading from form data or an uploaded file
51         if self.cleaned_data['data'] and import_method != ImportMethodChoices.DIRECT:
52             raise forms.ValidationError("Form data must be empty when uploading/selecting a file.")
53         if import_method == ImportMethodChoices.UPLOAD:
54             self.upload_file = 'upload_file'
55             file = self.files.get('upload_file')
56             data = file.read().decode('utf-8-sig')
57         elif import_method == ImportMethodChoices.DATA_FILE:
58             data = self.cleaned_data['data_file'].data_as_string
59         else:
60             data = self.cleaned_data['data']
61 
62         # Determine the data format
63         if self.cleaned_data['format'] == ImportFormatChoices.AUTO:
64             if self.cleaned_data['csv_delimiter'] != CSVDelimiterChoices.AUTO:
65                 # Specifying the CSV delimiter implies CSV format
66                 format = ImportFormatChoices.CSV
67             else:
68                 format = self._detect_format(data)
69         else:
70             format = self.cleaned_data['format']
71 
72         # Process data according to the selected format
73         if format == ImportFormatChoices.CSV:
74             delimiter = self.cleaned_data.get('csv_delimiter', CSVDelimiterChoices.AUTO)
75             self.cleaned_data['data'] = self._clean_csv(data, delimiter=delimiter)
76         elif format == ImportFormatChoices.JSON:
77             self.cleaned_data['data'] = self._clean_json(data)
78         elif format == ImportFormatChoices.YAML:
79             self.cleaned_data['data'] = self._clean_yaml(data)
80         else:
81             raise forms.ValidationError(f"Unknown data format: {format}")
82 
83     def _detect_format(self, data):
84         """
85         Attempt to automatically detect the format (CSV, JSON, or YAML) of the given data, or raise
86         a ValidationError.
87         """
88         try:
89             if data[0] in ('{', '['):
90                 return ImportFormatChoices.JSON
91             if data.startswith('---') or data.startswith('- '):
92                 return ImportFormatChoices.YAML
93             # Look for any of the CSV delimiters in the first line (ignoring the default 'auto' choice)
94             first_line = data.split('\n', 1)[0]
95             csv_delimiters = CSV_DELIMITERS.values()
96             if any(x in first_line for x in csv_delimiters):
97                 return ImportFormatChoices.CSV
98         except IndexError:
99             pass
100         raise forms.ValidationError({
101             'format': _('Unable to detect data format. Please specify.')
102         })
103 
104     def _clean_csv(self, data, delimiter=CSVDelimiterChoices.AUTO):
105         """
106         Clean CSV-formatted data. The first row will be treated as column headers.
107         """
108         # Determine the CSV dialect
109         if delimiter == CSVDelimiterChoices.AUTO:
110             # This uses a rough heuristic to detect the CSV dialect based on the presence of supported delimiting
111             # characters. If the data is malformed, we'll fall back to the default Excel dialect.
112             delimiters = ''.join(CSV_DELIMITERS.values())
113             try:
114                 dialect = csv.Sniffer().sniff(data.strip(), delimiters=delimiters)
115             except csv.Error:
116                 dialect = csv.excel
117         elif delimiter in (CSVDelimiterChoices.COMMA, CSVDelimiterChoices.SEMICOLON):
118             dialect = csv.excel
119             dialect.delimiter = delimiter
120         elif delimiter == CSVDelimiterChoices.TAB:
121             dialect = csv.excel_tab
122         else:
123             raise forms.ValidationError({
124                 'csv_delimiter': _('Invalid CSV delimiter'),
125             })
126 
127         stream = StringIO(data.strip())
128         reader = csv.reader(stream, dialect=dialect)
129         headers, records = parse_csv(reader)
130 
131         # Set CSV headers for reference by the model form
132         self._csv_headers = headers
133 
134         return records
135 
136     def _clean_json(self, data):
137         """
138         Clean JSON-formatted data. If only a single object is defined, it will be encapsulated as a list.
139         """
140         try:
141             data = json.loads(data)
142             # Accommodate for users entering single objects
143             if type(data) is not list:
144                 data = [data]
145             return data
146         except json.decoder.JSONDecodeError as err:
147             raise forms.ValidationError({
148                 self.data_field: f"Invalid JSON data: {err}"
149             })
150 
151     def _clean_yaml(self, data):
152         """
153         Clean YAML-formatted data. Data must be either
154           a) A single document comprising a list of dictionaries (each representing an object), or
155           b) Multiple documents, separated with the '---' token
156         """
157         records = []
158         try:
159             for data in yaml.load_all(data, Loader=yaml.SafeLoader):
160                 if type(data) is list:
161                     records.extend(data)
162                 elif type(data) is dict:
163                     records.append(data)
164                 else:
165                     raise forms.ValidationError({
166                         self.data_field: _(
167                             "Invalid YAML data. Data must be in the form of multiple documents, or a single document "
168                             "comprising a list of dictionaries."
169                         )
170                     })
171         except yaml.error.YAMLError as err:
172             raise forms.ValidationError({
173                 self.data_field: f"Invalid YAML data: {err}"
174             })
175 
176         return records
177
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/netbox/utilities/forms/bulk_import.py b/netbox/utilities/forms/bulk_import.py
--- a/netbox/utilities/forms/bulk_import.py
+++ b/netbox/utilities/forms/bulk_import.py
@@ -129,6 +129,7 @@
         headers, records = parse_csv(reader)
 
         # Set CSV headers for reference by the model form
+        headers.pop('id', None)
         self._csv_headers = headers
 
         return records
diff --git a/netbox/utilities/forms/forms.py b/netbox/utilities/forms/forms.py
--- a/netbox/utilities/forms/forms.py
+++ b/netbox/utilities/forms/forms.py
@@ -70,22 +70,24 @@
     """
     ModelForm used for the import of objects in CSV format.
     """
-    def __init__(self, *args, headers=None, fields=None, **kwargs):
-        headers = headers or {}
-        fields = fields or []
+    def __init__(self, *args, headers=None, **kwargs):
+        self.headers = headers or {}
         super().__init__(*args, **kwargs)
 
         # Modify the model form to accommodate any customized to_field_name properties
-        for field, to_field in headers.items():
+        for field, to_field in self.headers.items():
             if to_field is not None:
                 self.fields[field].to_field_name = to_field
 
-        # Omit any fields not specified (e.g. because the form is being used to
-        # updated rather than create objects)
-        if fields:
-            for field in list(self.fields.keys()):
-                if field not in fields:
-                    del self.fields[field]
+    def clean(self):
+        # Flag any invalid CSV headers
+        for header in self.headers:
+            if header not in self.fields:
+                raise forms.ValidationError(
+                    _("Unrecognized header: {name}").format(name=header)
+                )
+
+        return super().clean()
 
 
 class FilterForm(BootstrapMixin, forms.Form):

{"golden_diff": "diff --git a/netbox/utilities/forms/bulk_import.py b/netbox/utilities/forms/bulk_import.py\n--- a/netbox/utilities/forms/bulk_import.py\n+++ b/netbox/utilities/forms/bulk_import.py\n@@ -129,6 +129,7 @@\n         headers, records = parse_csv(reader)\n \n         # Set CSV headers for reference by the model form\n+        headers.pop('id', None)\n         self._csv_headers = headers\n \n         return records\ndiff --git a/netbox/utilities/forms/forms.py b/netbox/utilities/forms/forms.py\n--- a/netbox/utilities/forms/forms.py\n+++ b/netbox/utilities/forms/forms.py\n@@ -70,22 +70,24 @@\n     \"\"\"\n     ModelForm used for the import of objects in CSV format.\n     \"\"\"\n-    def __init__(self, *args, headers=None, fields=None, **kwargs):\n-        headers = headers or {}\n-        fields = fields or []\n+    def __init__(self, *args, headers=None, **kwargs):\n+        self.headers = headers or {}\n         super().__init__(*args, **kwargs)\n \n         # Modify the model form to accommodate any customized to_field_name properties\n-        for field, to_field in headers.items():\n+        for field, to_field in self.headers.items():\n             if to_field is not None:\n                 self.fields[field].to_field_name = to_field\n \n-        # Omit any fields not specified (e.g. because the form is being used to\n-        # updated rather than create objects)\n-        if fields:\n-            for field in list(self.fields.keys()):\n-                if field not in fields:\n-                    del self.fields[field]\n+    def clean(self):\n+        # Flag any invalid CSV headers\n+        for header in self.headers:\n+            if header not in self.fields:\n+                raise forms.ValidationError(\n+                    _(\"Unrecognized header: {name}\").format(name=header)\n+                )\n+\n+        return super().clean()\n \n \n class FilterForm(BootstrapMixin, forms.Form):\n", "issue": "Validate all headers/fields when bulk import objects\n### NetBox version\n\nv3.4.3\n\n### Python version\n\n3.8\n\n### Steps to Reproduce\n\n1. Open Prefix Bulk Import\r\n2. Specify all required field options and add some dummy data\r\n`prefix,status,testa,testb`\r\n`10.10.10.0/24,active,a,b`\r\n4. Press Import\r\n5. Import is done without any complains\r\n\n\n### Expected Behavior\n\nFields which are not stated in Field Options should throw an error, or a warning that some data was ignored.\r\n\n\n### Observed Behavior\n\nImport is done without any complains, Looks like all data was imported correct, although some fields were ignored\n", "before_files": [{"content": "import re\n\nfrom django import forms\nfrom django.utils.translation import gettext as _\nfrom .mixins import BootstrapMixin\n\n__all__ = (\n    'BulkEditForm',\n    'BulkRenameForm',\n    'ConfirmationForm',\n    'CSVModelForm',\n    'FilterForm',\n    'TableConfigForm',\n)\n\n\nclass ConfirmationForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    A generic confirmation form. The form is not valid unless the `confirm` field is checked.\n    \"\"\"\n    return_url = forms.CharField(\n        required=False,\n        widget=forms.HiddenInput()\n    )\n    confirm = forms.BooleanField(\n        required=True,\n        widget=forms.HiddenInput(),\n        initial=True\n    )\n\n\nclass BulkEditForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    Provides bulk edit support for objects.\n    \"\"\"\n    nullable_fields = ()\n\n\nclass BulkRenameForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    An extendable form to be used for renaming objects in bulk.\n    \"\"\"\n    find = forms.CharField(\n        strip=False\n    )\n    replace = forms.CharField(\n        strip=False,\n        required=False\n    )\n    use_regex = forms.BooleanField(\n        required=False,\n        initial=True,\n        label=_('Use regular expressions')\n    )\n\n    def clean(self):\n        super().clean()\n\n        # Validate regular expression in \"find\" field\n        if self.cleaned_data['use_regex']:\n            try:\n                re.compile(self.cleaned_data['find'])\n            except re.error:\n                raise forms.ValidationError({\n                    'find': \"Invalid regular expression\"\n                })\n\n\nclass CSVModelForm(forms.ModelForm):\n    \"\"\"\n    ModelForm used for the import of objects in CSV format.\n    \"\"\"\n    def __init__(self, *args, headers=None, fields=None, **kwargs):\n        headers = headers or {}\n        fields = fields or []\n        super().__init__(*args, **kwargs)\n\n        # Modify the model form to accommodate any customized to_field_name properties\n        for field, to_field in headers.items():\n            if to_field is not None:\n                self.fields[field].to_field_name = to_field\n\n        # Omit any fields not specified (e.g. because the form is being used to\n        # updated rather than create objects)\n        if fields:\n            for field in list(self.fields.keys()):\n                if field not in fields:\n                    del self.fields[field]\n\n\nclass FilterForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    Base Form class for FilterSet forms.\n    \"\"\"\n    q = forms.CharField(\n        required=False,\n        label=_('Search')\n    )\n\n\nclass TableConfigForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    Form for configuring user's table preferences.\n    \"\"\"\n    available_columns = forms.MultipleChoiceField(\n        choices=[],\n        required=False,\n        widget=forms.SelectMultiple(\n            attrs={'size': 10, 'class': 'form-select'}\n        ),\n        label=_('Available Columns')\n    )\n    columns = forms.MultipleChoiceField(\n        choices=[],\n        required=False,\n        widget=forms.SelectMultiple(\n            attrs={'size': 10, 'class': 'form-select'}\n        ),\n        label=_('Selected Columns')\n    )\n\n    def __init__(self, table, *args, **kwargs):\n        self.table = table\n\n        super().__init__(*args, **kwargs)\n\n        # Initialize columns field based on table attributes\n        self.fields['available_columns'].choices = table.available_columns\n        self.fields['columns'].choices = table.selected_columns\n\n    @property\n    def table_name(self):\n        return self.table.__class__.__name__\n", "path": "netbox/utilities/forms/forms.py"}, {"content": "import csv\nimport json\nfrom io import StringIO\n\nimport yaml\nfrom django import forms\nfrom django.utils.translation import gettext as _\n\nfrom core.forms.mixins import SyncedDataMixin\nfrom utilities.choices import CSVDelimiterChoices, ImportFormatChoices, ImportMethodChoices\nfrom utilities.constants import CSV_DELIMITERS\nfrom utilities.forms.utils import parse_csv\nfrom .mixins import BootstrapMixin\n\n\nclass BulkImportForm(BootstrapMixin, SyncedDataMixin, forms.Form):\n    import_method = forms.ChoiceField(\n        choices=ImportMethodChoices,\n        required=False\n    )\n    data = forms.CharField(\n        required=False,\n        widget=forms.Textarea(attrs={'class': 'font-monospace'}),\n        help_text=_(\"Enter object data in CSV, JSON or YAML format.\")\n    )\n    upload_file = forms.FileField(\n        label=_(\"Data file\"),\n        required=False\n    )\n    format = forms.ChoiceField(\n        choices=ImportFormatChoices,\n        initial=ImportFormatChoices.AUTO\n    )\n    csv_delimiter = forms.ChoiceField(\n        choices=CSVDelimiterChoices,\n        initial=CSVDelimiterChoices.AUTO,\n        label=_(\"CSV delimiter\"),\n        help_text=_(\"The character which delimits CSV fields. Applies only to CSV format.\"),\n        required=False\n    )\n\n    data_field = 'data'\n\n    def clean(self):\n        super().clean()\n\n        # Determine import method\n        import_method = self.cleaned_data.get('import_method') or ImportMethodChoices.DIRECT\n\n        # Determine whether we're reading from form data or an uploaded file\n        if self.cleaned_data['data'] and import_method != ImportMethodChoices.DIRECT:\n            raise forms.ValidationError(\"Form data must be empty when uploading/selecting a file.\")\n        if import_method == ImportMethodChoices.UPLOAD:\n            self.upload_file = 'upload_file'\n            file = self.files.get('upload_file')\n            data = file.read().decode('utf-8-sig')\n        elif import_method == ImportMethodChoices.DATA_FILE:\n            data = self.cleaned_data['data_file'].data_as_string\n        else:\n            data = self.cleaned_data['data']\n\n        # Determine the data format\n        if self.cleaned_data['format'] == ImportFormatChoices.AUTO:\n            if self.cleaned_data['csv_delimiter'] != CSVDelimiterChoices.AUTO:\n                # Specifying the CSV delimiter implies CSV format\n                format = ImportFormatChoices.CSV\n            else:\n                format = self._detect_format(data)\n        else:\n            format = self.cleaned_data['format']\n\n        # Process data according to the selected format\n        if format == ImportFormatChoices.CSV:\n            delimiter = self.cleaned_data.get('csv_delimiter', CSVDelimiterChoices.AUTO)\n            self.cleaned_data['data'] = self._clean_csv(data, delimiter=delimiter)\n        elif format == ImportFormatChoices.JSON:\n            self.cleaned_data['data'] = self._clean_json(data)\n        elif format == ImportFormatChoices.YAML:\n            self.cleaned_data['data'] = self._clean_yaml(data)\n        else:\n            raise forms.ValidationError(f\"Unknown data format: {format}\")\n\n    def _detect_format(self, data):\n        \"\"\"\n        Attempt to automatically detect the format (CSV, JSON, or YAML) of the given data, or raise\n        a ValidationError.\n        \"\"\"\n        try:\n            if data[0] in ('{', '['):\n                return ImportFormatChoices.JSON\n            if data.startswith('---') or data.startswith('- '):\n                return ImportFormatChoices.YAML\n            # Look for any of the CSV delimiters in the first line (ignoring the default 'auto' choice)\n            first_line = data.split('\\n', 1)[0]\n            csv_delimiters = CSV_DELIMITERS.values()\n            if any(x in first_line for x in csv_delimiters):\n                return ImportFormatChoices.CSV\n        except IndexError:\n            pass\n        raise forms.ValidationError({\n            'format': _('Unable to detect data format. Please specify.')\n        })\n\n    def _clean_csv(self, data, delimiter=CSVDelimiterChoices.AUTO):\n        \"\"\"\n        Clean CSV-formatted data. The first row will be treated as column headers.\n        \"\"\"\n        # Determine the CSV dialect\n        if delimiter == CSVDelimiterChoices.AUTO:\n            # This uses a rough heuristic to detect the CSV dialect based on the presence of supported delimiting\n            # characters. If the data is malformed, we'll fall back to the default Excel dialect.\n            delimiters = ''.join(CSV_DELIMITERS.values())\n            try:\n                dialect = csv.Sniffer().sniff(data.strip(), delimiters=delimiters)\n            except csv.Error:\n                dialect = csv.excel\n        elif delimiter in (CSVDelimiterChoices.COMMA, CSVDelimiterChoices.SEMICOLON):\n            dialect = csv.excel\n            dialect.delimiter = delimiter\n        elif delimiter == CSVDelimiterChoices.TAB:\n            dialect = csv.excel_tab\n        else:\n            raise forms.ValidationError({\n                'csv_delimiter': _('Invalid CSV delimiter'),\n            })\n\n        stream = StringIO(data.strip())\n        reader = csv.reader(stream, dialect=dialect)\n        headers, records = parse_csv(reader)\n\n        # Set CSV headers for reference by the model form\n        self._csv_headers = headers\n\n        return records\n\n    def _clean_json(self, data):\n        \"\"\"\n        Clean JSON-formatted data. If only a single object is defined, it will be encapsulated as a list.\n        \"\"\"\n        try:\n            data = json.loads(data)\n            # Accommodate for users entering single objects\n            if type(data) is not list:\n                data = [data]\n            return data\n        except json.decoder.JSONDecodeError as err:\n            raise forms.ValidationError({\n                self.data_field: f\"Invalid JSON data: {err}\"\n            })\n\n    def _clean_yaml(self, data):\n        \"\"\"\n        Clean YAML-formatted data. Data must be either\n          a) A single document comprising a list of dictionaries (each representing an object), or\n          b) Multiple documents, separated with the '---' token\n        \"\"\"\n        records = []\n        try:\n            for data in yaml.load_all(data, Loader=yaml.SafeLoader):\n                if type(data) is list:\n                    records.extend(data)\n                elif type(data) is dict:\n                    records.append(data)\n                else:\n                    raise forms.ValidationError({\n                        self.data_field: _(\n                            \"Invalid YAML data. Data must be in the form of multiple documents, or a single document \"\n                            \"comprising a list of dictionaries.\"\n                        )\n                    })\n        except yaml.error.YAMLError as err:\n            raise forms.ValidationError({\n                self.data_field: f\"Invalid YAML data: {err}\"\n            })\n\n        return records\n", "path": "netbox/utilities/forms/bulk_import.py"}], "after_files": [{"content": "import re\n\nfrom django import forms\nfrom django.utils.translation import gettext as _\nfrom .mixins import BootstrapMixin\n\n__all__ = (\n    'BulkEditForm',\n    'BulkRenameForm',\n    'ConfirmationForm',\n    'CSVModelForm',\n    'FilterForm',\n    'TableConfigForm',\n)\n\n\nclass ConfirmationForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    A generic confirmation form. The form is not valid unless the `confirm` field is checked.\n    \"\"\"\n    return_url = forms.CharField(\n        required=False,\n        widget=forms.HiddenInput()\n    )\n    confirm = forms.BooleanField(\n        required=True,\n        widget=forms.HiddenInput(),\n        initial=True\n    )\n\n\nclass BulkEditForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    Provides bulk edit support for objects.\n    \"\"\"\n    nullable_fields = ()\n\n\nclass BulkRenameForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    An extendable form to be used for renaming objects in bulk.\n    \"\"\"\n    find = forms.CharField(\n        strip=False\n    )\n    replace = forms.CharField(\n        strip=False,\n        required=False\n    )\n    use_regex = forms.BooleanField(\n        required=False,\n        initial=True,\n        label=_('Use regular expressions')\n    )\n\n    def clean(self):\n        super().clean()\n\n        # Validate regular expression in \"find\" field\n        if self.cleaned_data['use_regex']:\n            try:\n                re.compile(self.cleaned_data['find'])\n            except re.error:\n                raise forms.ValidationError({\n                    'find': \"Invalid regular expression\"\n                })\n\n\nclass CSVModelForm(forms.ModelForm):\n    \"\"\"\n    ModelForm used for the import of objects in CSV format.\n    \"\"\"\n    def __init__(self, *args, headers=None, **kwargs):\n        self.headers = headers or {}\n        super().__init__(*args, **kwargs)\n\n        # Modify the model form to accommodate any customized to_field_name properties\n        for field, to_field in self.headers.items():\n            if to_field is not None:\n                self.fields[field].to_field_name = to_field\n\n    def clean(self):\n        # Flag any invalid CSV headers\n        for header in self.headers:\n            if header not in self.fields:\n                raise forms.ValidationError(\n                    _(\"Unrecognized header: {name}\").format(name=header)\n                )\n\n        return super().clean()\n\n\nclass FilterForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    Base Form class for FilterSet forms.\n    \"\"\"\n    q = forms.CharField(\n        required=False,\n        label=_('Search')\n    )\n\n\nclass TableConfigForm(BootstrapMixin, forms.Form):\n    \"\"\"\n    Form for configuring user's table preferences.\n    \"\"\"\n    available_columns = forms.MultipleChoiceField(\n        choices=[],\n        required=False,\n        widget=forms.SelectMultiple(\n            attrs={'size': 10, 'class': 'form-select'}\n        ),\n        label=_('Available Columns')\n    )\n    columns = forms.MultipleChoiceField(\n        choices=[],\n        required=False,\n        widget=forms.SelectMultiple(\n            attrs={'size': 10, 'class': 'form-select'}\n        ),\n        label=_('Selected Columns')\n    )\n\n    def __init__(self, table, *args, **kwargs):\n        self.table = table\n\n        super().__init__(*args, **kwargs)\n\n        # Initialize columns field based on table attributes\n        self.fields['available_columns'].choices = table.available_columns\n        self.fields['columns'].choices = table.selected_columns\n\n    @property\n    def table_name(self):\n        return self.table.__class__.__name__\n", "path": "netbox/utilities/forms/forms.py"}, {"content": "import csv\nimport json\nfrom io import StringIO\n\nimport yaml\nfrom django import forms\nfrom django.utils.translation import gettext as _\n\nfrom core.forms.mixins import SyncedDataMixin\nfrom utilities.choices import CSVDelimiterChoices, ImportFormatChoices, ImportMethodChoices\nfrom utilities.constants import CSV_DELIMITERS\nfrom utilities.forms.utils import parse_csv\nfrom .mixins import BootstrapMixin\n\n\nclass BulkImportForm(BootstrapMixin, SyncedDataMixin, forms.Form):\n    import_method = forms.ChoiceField(\n        choices=ImportMethodChoices,\n        required=False\n    )\n    data = forms.CharField(\n        required=False,\n        widget=forms.Textarea(attrs={'class': 'font-monospace'}),\n        help_text=_(\"Enter object data in CSV, JSON or YAML format.\")\n    )\n    upload_file = forms.FileField(\n        label=_(\"Data file\"),\n        required=False\n    )\n    format = forms.ChoiceField(\n        choices=ImportFormatChoices,\n        initial=ImportFormatChoices.AUTO\n    )\n    csv_delimiter = forms.ChoiceField(\n        choices=CSVDelimiterChoices,\n        initial=CSVDelimiterChoices.AUTO,\n        label=_(\"CSV delimiter\"),\n        help_text=_(\"The character which delimits CSV fields. Applies only to CSV format.\"),\n        required=False\n    )\n\n    data_field = 'data'\n\n    def clean(self):\n        super().clean()\n\n        # Determine import method\n        import_method = self.cleaned_data.get('import_method') or ImportMethodChoices.DIRECT\n\n        # Determine whether we're reading from form data or an uploaded file\n        if self.cleaned_data['data'] and import_method != ImportMethodChoices.DIRECT:\n            raise forms.ValidationError(\"Form data must be empty when uploading/selecting a file.\")\n        if import_method == ImportMethodChoices.UPLOAD:\n            self.upload_file = 'upload_file'\n            file = self.files.get('upload_file')\n            data = file.read().decode('utf-8-sig')\n        elif import_method == ImportMethodChoices.DATA_FILE:\n            data = self.cleaned_data['data_file'].data_as_string\n        else:\n            data = self.cleaned_data['data']\n\n        # Determine the data format\n        if self.cleaned_data['format'] == ImportFormatChoices.AUTO:\n            if self.cleaned_data['csv_delimiter'] != CSVDelimiterChoices.AUTO:\n                # Specifying the CSV delimiter implies CSV format\n                format = ImportFormatChoices.CSV\n            else:\n                format = self._detect_format(data)\n        else:\n            format = self.cleaned_data['format']\n\n        # Process data according to the selected format\n        if format == ImportFormatChoices.CSV:\n            delimiter = self.cleaned_data.get('csv_delimiter', CSVDelimiterChoices.AUTO)\n            self.cleaned_data['data'] = self._clean_csv(data, delimiter=delimiter)\n        elif format == ImportFormatChoices.JSON:\n            self.cleaned_data['data'] = self._clean_json(data)\n        elif format == ImportFormatChoices.YAML:\n            self.cleaned_data['data'] = self._clean_yaml(data)\n        else:\n            raise forms.ValidationError(f\"Unknown data format: {format}\")\n\n    def _detect_format(self, data):\n        \"\"\"\n        Attempt to automatically detect the format (CSV, JSON, or YAML) of the given data, or raise\n        a ValidationError.\n        \"\"\"\n        try:\n            if data[0] in ('{', '['):\n                return ImportFormatChoices.JSON\n            if data.startswith('---') or data.startswith('- '):\n                return ImportFormatChoices.YAML\n            # Look for any of the CSV delimiters in the first line (ignoring the default 'auto' choice)\n            first_line = data.split('\\n', 1)[0]\n            csv_delimiters = CSV_DELIMITERS.values()\n            if any(x in first_line for x in csv_delimiters):\n                return ImportFormatChoices.CSV\n        except IndexError:\n            pass\n        raise forms.ValidationError({\n            'format': _('Unable to detect data format. Please specify.')\n        })\n\n    def _clean_csv(self, data, delimiter=CSVDelimiterChoices.AUTO):\n        \"\"\"\n        Clean CSV-formatted data. The first row will be treated as column headers.\n        \"\"\"\n        # Determine the CSV dialect\n        if delimiter == CSVDelimiterChoices.AUTO:\n            # This uses a rough heuristic to detect the CSV dialect based on the presence of supported delimiting\n            # characters. If the data is malformed, we'll fall back to the default Excel dialect.\n            delimiters = ''.join(CSV_DELIMITERS.values())\n            try:\n                dialect = csv.Sniffer().sniff(data.strip(), delimiters=delimiters)\n            except csv.Error:\n                dialect = csv.excel\n        elif delimiter in (CSVDelimiterChoices.COMMA, CSVDelimiterChoices.SEMICOLON):\n            dialect = csv.excel\n            dialect.delimiter = delimiter\n        elif delimiter == CSVDelimiterChoices.TAB:\n            dialect = csv.excel_tab\n        else:\n            raise forms.ValidationError({\n                'csv_delimiter': _('Invalid CSV delimiter'),\n            })\n\n        stream = StringIO(data.strip())\n        reader = csv.reader(stream, dialect=dialect)\n        headers, records = parse_csv(reader)\n\n        # Set CSV headers for reference by the model form\n        headers.pop('id', None)\n        self._csv_headers = headers\n\n        return records\n\n    def _clean_json(self, data):\n        \"\"\"\n        Clean JSON-formatted data. If only a single object is defined, it will be encapsulated as a list.\n        \"\"\"\n        try:\n            data = json.loads(data)\n            # Accommodate for users entering single objects\n            if type(data) is not list:\n                data = [data]\n            return data\n        except json.decoder.JSONDecodeError as err:\n            raise forms.ValidationError({\n                self.data_field: f\"Invalid JSON data: {err}\"\n            })\n\n    def _clean_yaml(self, data):\n        \"\"\"\n        Clean YAML-formatted data. Data must be either\n          a) A single document comprising a list of dictionaries (each representing an object), or\n          b) Multiple documents, separated with the '---' token\n        \"\"\"\n        records = []\n        try:\n            for data in yaml.load_all(data, Loader=yaml.SafeLoader):\n                if type(data) is list:\n                    records.extend(data)\n                elif type(data) is dict:\n                    records.append(data)\n                else:\n                    raise forms.ValidationError({\n                        self.data_field: _(\n                            \"Invalid YAML data. Data must be in the form of multiple documents, or a single document \"\n                            \"comprising a list of dictionaries.\"\n                        )\n                    })\n        except yaml.error.YAMLError as err:\n            raise forms.ValidationError({\n                self.data_field: f\"Invalid YAML data: {err}\"\n            })\n\n        return records\n", "path": "netbox/utilities/forms/bulk_import.py"}]}