We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[BUG] Error if upload file with not valid symbol
Here is example of such file: https://www.kaggle.com/datasets/fedesoriano/covid19-effect-on-liver-cancer-prediction-dataset?select=covid-liver.csv
Column `Type_of_incidental_finding` contains not valid symbols. 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `mindsdb/integrations/handlers/file_handler/file_handler.py`
Content:
```
1 from io import BytesIO, StringIO
2 import os
3 import csv
4 import json
5 import codecs
6 import traceback
7 import tempfile
8 from urllib.parse import urlparse
9 
10 import requests
11 import pandas as pd
12 from charset_normalizer import from_bytes
13 
14 from mindsdb_sql import parse_sql
15 from mindsdb_sql.parser.ast.base import ASTNode
16 from mindsdb_sql.parser.ast import DropTables, Select
17 
18 from mindsdb.api.mysql.mysql_proxy.utilities.sql import query_df
19 from mindsdb.integrations.libs.base_handler import DatabaseHandler
20 from mindsdb.integrations.libs.response import (
21     HandlerStatusResponse as StatusResponse,
22     HandlerResponse as Response,
23     RESPONSE_TYPE
24 )
25 
26 
27 def clean_row(row):
28     n_row = []
29     for cell in row:
30         if str(cell) in ['', ' ', '  ', 'NaN', 'nan', 'NA']:
31             n_row.append(None)
32         else:
33             n_row.append(cell)
34 
35     return n_row
36 
37 
38 class FileHandler(DatabaseHandler):
39     """
40     Handler for files
41     """
42     name = 'files'
43 
44     def __init__(self, name=None, file_storage=None, connection_data={}, file_controller=None):
45         super().__init__(name)
46         self.parser = parse_sql
47         self.fs_store = file_storage
48         self.custom_parser = connection_data.get('custom_parser', None)
49         self.clean_rows = connection_data.get('clean_rows', True)
50         self.file_controller = file_controller
51 
52     def connect(self, **kwargs):
53         return
54 
55     def disconnect(self, **kwargs):
56         return
57 
58     def check_connection(self) -> StatusResponse:
59         return StatusResponse(True)
60 
61     def query(self, query: ASTNode) -> Response:
62         if type(query) == DropTables:
63             for table_identifier in query.tables:
64                 if len(table_identifier.parts) == 2 and table_identifier.parts[0] != self.name:
65                     return Response(
66                         RESPONSE_TYPE.ERROR,
67                         error_message=f"Can't delete table from database '{table_identifier.parts[0]}'"
68                     )
69                 table_name = table_identifier.parts[-1]
70                 try:
71                     self.file_controller.delete_file(table_name)
72                 except Exception as e:
73                     return Response(
74                         RESPONSE_TYPE.ERROR,
75                         error_message=f"Can't delete table '{table_name}': {e}"
76                     )
77             return Response(RESPONSE_TYPE.OK)
78         elif type(query) == Select:
79             table_name = query.from_table.parts[-1]
80             file_path = self.file_controller.get_file_path(table_name)
81             df, _columns = self._handle_source(file_path, self.clean_rows, self.custom_parser)
82             result_df = query_df(df, query)
83             return Response(
84                 RESPONSE_TYPE.TABLE,
85                 data_frame=result_df
86             )
87         else:
88             return Response(
89                 RESPONSE_TYPE.ERROR,
90                 error_message="Only 'select' and 'drop' queries allowed for files"
91             )
92 
93     def native_query(self, query: str) -> Response:
94         ast = self.parser(query, dialect='mindsdb')
95         return self.query(ast)
96 
97     @staticmethod
98     def _handle_source(file_path, clean_rows=True, custom_parser=None):
99         # get file data io, format and dialect
100         data, fmt, dialect = FileHandler._get_data_io(file_path)
101         data.seek(0)  # make sure we are at 0 in file pointer
102 
103         if custom_parser:
104             header, file_data = custom_parser(data, fmt)
105 
106         elif fmt == 'csv':
107             csv_reader = list(csv.reader(data, dialect))
108             header = csv_reader[0]
109             file_data = csv_reader[1:]
110 
111         elif fmt in ['xlsx', 'xls']:
112             data.seek(0)
113             df = pd.read_excel(data)
114             header = df.columns.values.tolist()
115             file_data = df.values.tolist()
116 
117         elif fmt == 'json':
118             data.seek(0)
119             json_doc = json.loads(data.read())
120             df = pd.json_normalize(json_doc, max_level=0)
121             header = df.columns.values.tolist()
122             file_data = df.values.tolist()
123 
124         else:
125             raise ValueError('Could not load file into any format, supported formats are csv, json, xls, xlsx')
126 
127         if clean_rows:
128             file_list_data = [clean_row(row) for row in file_data]
129         else:
130             file_list_data = file_data
131 
132         header = [x.strip() for x in header]
133         col_map = dict((col, col) for col in header)
134         return pd.DataFrame(file_list_data, columns=header), col_map
135 
136     @staticmethod
137     def _get_data_io(file_path):
138         """
139         This gets a file either url or local file and defines what the format is as well as dialect
140         :param file: file path or url
141         :return: data_io, format, dialect
142         """
143 
144         ############
145         # get file as io object
146         ############
147 
148         # file_path = self._get_file_path()
149 
150         data = BytesIO()
151 
152         try:
153             with open(file_path, 'rb') as fp:
154                 data = BytesIO(fp.read())
155         except Exception as e:
156             error = 'Could not load file, possible exception : {exception}'.format(exception=e)
157             print(error)
158             raise ValueError(error)
159 
160         dialect = None
161 
162         ############
163         # check for file type
164         ############
165 
166         # try to guess if its an excel file
167         xlsx_sig = b'\x50\x4B\x05\06'
168         # xlsx_sig2 = b'\x50\x4B\x03\x04'
169         xls_sig = b'\x09\x08\x10\x00\x00\x06\x05\x00'
170 
171         # different whence, offset, size for different types
172         excel_meta = [('xls', 0, 512, 8), ('xlsx', 2, -22, 4)]
173 
174         for filename, whence, offset, size in excel_meta:
175 
176             try:
177                 data.seek(offset, whence)  # Seek to the offset.
178                 bytes = data.read(size)  # Capture the specified number of bytes.
179                 data.seek(0)
180                 codecs.getencoder('hex')(bytes)
181 
182                 if bytes == xls_sig:
183                     return data, 'xls', dialect
184                 elif bytes == xlsx_sig:
185                     return data, 'xlsx', dialect
186 
187             except Exception:
188                 data.seek(0)
189 
190         # if not excel it can be a json file or a CSV, convert from binary to stringio
191 
192         byte_str = data.read()
193         # Move it to StringIO
194         try:
195             # Handle Microsoft's BOM "special" UTF-8 encoding
196             if byte_str.startswith(codecs.BOM_UTF8):
197                 data = StringIO(byte_str.decode('utf-8-sig'))
198             else:
199                 file_encoding_meta = from_bytes(
200                     byte_str[:32 * 1024],
201                     steps=32,           # Number of steps/block to extract from my_byte_str
202                     chunk_size=1024,    # Set block size of each extraction)
203                     explain=False
204                 )
205                 best_meta = file_encoding_meta.best()
206                 if best_meta is not None:
207                     encoding = file_encoding_meta.best().encoding
208                 else:
209                     encoding = 'utf-8'
210                 data = StringIO(byte_str.decode(encoding))
211         except Exception:
212             print(traceback.format_exc())
213             print('Could not load into string')
214 
215         # see if its JSON
216         buffer = data.read(100)
217         data.seek(0)
218         text = buffer.strip()
219         # analyze first n characters
220         if len(text) > 0:
221             text = text.strip()
222             # it it looks like a json, then try to parse it
223             if text.startswith('{') or text.startswith('['):
224                 try:
225                     json.loads(data.read())
226                     data.seek(0)
227                     return data, 'json', dialect
228                 except Exception:
229                     data.seek(0)
230                     return data, None, dialect
231 
232         # lets try to figure out if its a csv
233         try:
234             dialect = FileHandler._get_csv_dialect(data)
235             if dialect:
236                 return data, 'csv', dialect
237             return data, None, dialect
238         except Exception:
239             data.seek(0)
240             print('Could not detect format for this file')
241             print(traceback.format_exc())
242             # No file type identified
243             return data, None, dialect
244 
245     @staticmethod
246     def _get_file_path(path) -> str:
247         try:
248             is_url = urlparse(path).scheme in ('http', 'https')
249         except Exception:
250             is_url = False
251         if is_url:
252             path = FileHandler._fetch_url(path)
253         return path
254 
255     @staticmethod
256     def _get_csv_dialect(buffer) -> csv.Dialect:
257         sample = buffer.read(128 * 1024)
258         buffer.seek(0)
259         try:
260             accepted_csv_delimiters = [',', '\t', ';']
261             dialect = csv.Sniffer().sniff(sample, delimiters=accepted_csv_delimiters)
262         except csv.Error:
263             dialect = None
264         return dialect
265 
266     @staticmethod
267     def _fetch_url(url: str) -> str:
268         temp_dir = tempfile.mkdtemp(prefix='mindsdb_file_url_')
269         try:
270             r = requests.get(url, stream=True)
271             if r.status_code == 200:
272                 with open(os.path.join(temp_dir, 'file'), 'wb') as f:
273                     for chunk in r:
274                         f.write(chunk)
275             else:
276                 raise Exception(f'Responce status code is {r.status_code}')
277         except Exception as e:
278             print(f'Error during getting {url}')
279             print(e)
280             raise
281         return os.path.join(temp_dir, 'file')
282 
283     def get_tables(self) -> Response:
284         """
285         List all files
286         """
287         files_meta = self.file_controller.get_files()
288         data = [{
289             'TABLE_NAME': x['name'],
290             'TABLE_ROWS': x['row_count']
291         } for x in files_meta]
292         return Response(
293             RESPONSE_TYPE.TABLE,
294             data_frame=pd.DataFrame(data)
295         )
296 
297     def get_columns(self, table_name) -> Response:
298         file_meta = self.file_controller.get_file_meta(table_name)
299         result = Response(
300             RESPONSE_TYPE.TABLE,
301             data_frame=pd.DataFrame([
302                 {
303                     'Field': x.strip(),
304                     'Type': 'str'
305                 } for x in file_meta['columns']
306             ])
307         )
308         return result
309
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/mindsdb/integrations/handlers/file_handler/file_handler.py b/mindsdb/integrations/handlers/file_handler/file_handler.py
--- a/mindsdb/integrations/handlers/file_handler/file_handler.py
+++ b/mindsdb/integrations/handlers/file_handler/file_handler.py
@@ -203,11 +203,13 @@
                     explain=False
                 )
                 best_meta = file_encoding_meta.best()
+                errors = 'strict'
                 if best_meta is not None:
                     encoding = file_encoding_meta.best().encoding
                 else:
                     encoding = 'utf-8'
-                data = StringIO(byte_str.decode(encoding))
+                    errors = 'replace'
+                data = StringIO(byte_str.decode(encoding, errors))
         except Exception:
             print(traceback.format_exc())
             print('Could not load into string')

{"golden_diff": "diff --git a/mindsdb/integrations/handlers/file_handler/file_handler.py b/mindsdb/integrations/handlers/file_handler/file_handler.py\n--- a/mindsdb/integrations/handlers/file_handler/file_handler.py\n+++ b/mindsdb/integrations/handlers/file_handler/file_handler.py\n@@ -203,11 +203,13 @@\n                     explain=False\n                 )\n                 best_meta = file_encoding_meta.best()\n+                errors = 'strict'\n                 if best_meta is not None:\n                     encoding = file_encoding_meta.best().encoding\n                 else:\n                     encoding = 'utf-8'\n-                data = StringIO(byte_str.decode(encoding))\n+                    errors = 'replace'\n+                data = StringIO(byte_str.decode(encoding, errors))\n         except Exception:\n             print(traceback.format_exc())\n             print('Could not load into string')\n", "issue": "[BUG] Error if upload file with not valid symbol\nHere is example of such file: https://www.kaggle.com/datasets/fedesoriano/covid19-effect-on-liver-cancer-prediction-dataset?select=covid-liver.csv\r\nColumn `Type_of_incidental_finding` contains not valid symbols. \n", "before_files": [{"content": "from io import BytesIO, StringIO\nimport os\nimport csv\nimport json\nimport codecs\nimport traceback\nimport tempfile\nfrom urllib.parse import urlparse\n\nimport requests\nimport pandas as pd\nfrom charset_normalizer import from_bytes\n\nfrom mindsdb_sql import parse_sql\nfrom mindsdb_sql.parser.ast.base import ASTNode\nfrom mindsdb_sql.parser.ast import DropTables, Select\n\nfrom mindsdb.api.mysql.mysql_proxy.utilities.sql import query_df\nfrom mindsdb.integrations.libs.base_handler import DatabaseHandler\nfrom mindsdb.integrations.libs.response import (\n    HandlerStatusResponse as StatusResponse,\n    HandlerResponse as Response,\n    RESPONSE_TYPE\n)\n\n\ndef clean_row(row):\n    n_row = []\n    for cell in row:\n        if str(cell) in ['', ' ', '  ', 'NaN', 'nan', 'NA']:\n            n_row.append(None)\n        else:\n            n_row.append(cell)\n\n    return n_row\n\n\nclass FileHandler(DatabaseHandler):\n    \"\"\"\n    Handler for files\n    \"\"\"\n    name = 'files'\n\n    def __init__(self, name=None, file_storage=None, connection_data={}, file_controller=None):\n        super().__init__(name)\n        self.parser = parse_sql\n        self.fs_store = file_storage\n        self.custom_parser = connection_data.get('custom_parser', None)\n        self.clean_rows = connection_data.get('clean_rows', True)\n        self.file_controller = file_controller\n\n    def connect(self, **kwargs):\n        return\n\n    def disconnect(self, **kwargs):\n        return\n\n    def check_connection(self) -> StatusResponse:\n        return StatusResponse(True)\n\n    def query(self, query: ASTNode) -> Response:\n        if type(query) == DropTables:\n            for table_identifier in query.tables:\n                if len(table_identifier.parts) == 2 and table_identifier.parts[0] != self.name:\n                    return Response(\n                        RESPONSE_TYPE.ERROR,\n                        error_message=f\"Can't delete table from database '{table_identifier.parts[0]}'\"\n                    )\n                table_name = table_identifier.parts[-1]\n                try:\n                    self.file_controller.delete_file(table_name)\n                except Exception as e:\n                    return Response(\n                        RESPONSE_TYPE.ERROR,\n                        error_message=f\"Can't delete table '{table_name}': {e}\"\n                    )\n            return Response(RESPONSE_TYPE.OK)\n        elif type(query) == Select:\n            table_name = query.from_table.parts[-1]\n            file_path = self.file_controller.get_file_path(table_name)\n            df, _columns = self._handle_source(file_path, self.clean_rows, self.custom_parser)\n            result_df = query_df(df, query)\n            return Response(\n                RESPONSE_TYPE.TABLE,\n                data_frame=result_df\n            )\n        else:\n            return Response(\n                RESPONSE_TYPE.ERROR,\n                error_message=\"Only 'select' and 'drop' queries allowed for files\"\n            )\n\n    def native_query(self, query: str) -> Response:\n        ast = self.parser(query, dialect='mindsdb')\n        return self.query(ast)\n\n    @staticmethod\n    def _handle_source(file_path, clean_rows=True, custom_parser=None):\n        # get file data io, format and dialect\n        data, fmt, dialect = FileHandler._get_data_io(file_path)\n        data.seek(0)  # make sure we are at 0 in file pointer\n\n        if custom_parser:\n            header, file_data = custom_parser(data, fmt)\n\n        elif fmt == 'csv':\n            csv_reader = list(csv.reader(data, dialect))\n            header = csv_reader[0]\n            file_data = csv_reader[1:]\n\n        elif fmt in ['xlsx', 'xls']:\n            data.seek(0)\n            df = pd.read_excel(data)\n            header = df.columns.values.tolist()\n            file_data = df.values.tolist()\n\n        elif fmt == 'json':\n            data.seek(0)\n            json_doc = json.loads(data.read())\n            df = pd.json_normalize(json_doc, max_level=0)\n            header = df.columns.values.tolist()\n            file_data = df.values.tolist()\n\n        else:\n            raise ValueError('Could not load file into any format, supported formats are csv, json, xls, xlsx')\n\n        if clean_rows:\n            file_list_data = [clean_row(row) for row in file_data]\n        else:\n            file_list_data = file_data\n\n        header = [x.strip() for x in header]\n        col_map = dict((col, col) for col in header)\n        return pd.DataFrame(file_list_data, columns=header), col_map\n\n    @staticmethod\n    def _get_data_io(file_path):\n        \"\"\"\n        This gets a file either url or local file and defines what the format is as well as dialect\n        :param file: file path or url\n        :return: data_io, format, dialect\n        \"\"\"\n\n        ############\n        # get file as io object\n        ############\n\n        # file_path = self._get_file_path()\n\n        data = BytesIO()\n\n        try:\n            with open(file_path, 'rb') as fp:\n                data = BytesIO(fp.read())\n        except Exception as e:\n            error = 'Could not load file, possible exception : {exception}'.format(exception=e)\n            print(error)\n            raise ValueError(error)\n\n        dialect = None\n\n        ############\n        # check for file type\n        ############\n\n        # try to guess if its an excel file\n        xlsx_sig = b'\\x50\\x4B\\x05\\06'\n        # xlsx_sig2 = b'\\x50\\x4B\\x03\\x04'\n        xls_sig = b'\\x09\\x08\\x10\\x00\\x00\\x06\\x05\\x00'\n\n        # different whence, offset, size for different types\n        excel_meta = [('xls', 0, 512, 8), ('xlsx', 2, -22, 4)]\n\n        for filename, whence, offset, size in excel_meta:\n\n            try:\n                data.seek(offset, whence)  # Seek to the offset.\n                bytes = data.read(size)  # Capture the specified number of bytes.\n                data.seek(0)\n                codecs.getencoder('hex')(bytes)\n\n                if bytes == xls_sig:\n                    return data, 'xls', dialect\n                elif bytes == xlsx_sig:\n                    return data, 'xlsx', dialect\n\n            except Exception:\n                data.seek(0)\n\n        # if not excel it can be a json file or a CSV, convert from binary to stringio\n\n        byte_str = data.read()\n        # Move it to StringIO\n        try:\n            # Handle Microsoft's BOM \"special\" UTF-8 encoding\n            if byte_str.startswith(codecs.BOM_UTF8):\n                data = StringIO(byte_str.decode('utf-8-sig'))\n            else:\n                file_encoding_meta = from_bytes(\n                    byte_str[:32 * 1024],\n                    steps=32,           # Number of steps/block to extract from my_byte_str\n                    chunk_size=1024,    # Set block size of each extraction)\n                    explain=False\n                )\n                best_meta = file_encoding_meta.best()\n                if best_meta is not None:\n                    encoding = file_encoding_meta.best().encoding\n                else:\n                    encoding = 'utf-8'\n                data = StringIO(byte_str.decode(encoding))\n        except Exception:\n            print(traceback.format_exc())\n            print('Could not load into string')\n\n        # see if its JSON\n        buffer = data.read(100)\n        data.seek(0)\n        text = buffer.strip()\n        # analyze first n characters\n        if len(text) > 0:\n            text = text.strip()\n            # it it looks like a json, then try to parse it\n            if text.startswith('{') or text.startswith('['):\n                try:\n                    json.loads(data.read())\n                    data.seek(0)\n                    return data, 'json', dialect\n                except Exception:\n                    data.seek(0)\n                    return data, None, dialect\n\n        # lets try to figure out if its a csv\n        try:\n            dialect = FileHandler._get_csv_dialect(data)\n            if dialect:\n                return data, 'csv', dialect\n            return data, None, dialect\n        except Exception:\n            data.seek(0)\n            print('Could not detect format for this file')\n            print(traceback.format_exc())\n            # No file type identified\n            return data, None, dialect\n\n    @staticmethod\n    def _get_file_path(path) -> str:\n        try:\n            is_url = urlparse(path).scheme in ('http', 'https')\n        except Exception:\n            is_url = False\n        if is_url:\n            path = FileHandler._fetch_url(path)\n        return path\n\n    @staticmethod\n    def _get_csv_dialect(buffer) -> csv.Dialect:\n        sample = buffer.read(128 * 1024)\n        buffer.seek(0)\n        try:\n            accepted_csv_delimiters = [',', '\\t', ';']\n            dialect = csv.Sniffer().sniff(sample, delimiters=accepted_csv_delimiters)\n        except csv.Error:\n            dialect = None\n        return dialect\n\n    @staticmethod\n    def _fetch_url(url: str) -> str:\n        temp_dir = tempfile.mkdtemp(prefix='mindsdb_file_url_')\n        try:\n            r = requests.get(url, stream=True)\n            if r.status_code == 200:\n                with open(os.path.join(temp_dir, 'file'), 'wb') as f:\n                    for chunk in r:\n                        f.write(chunk)\n            else:\n                raise Exception(f'Responce status code is {r.status_code}')\n        except Exception as e:\n            print(f'Error during getting {url}')\n            print(e)\n            raise\n        return os.path.join(temp_dir, 'file')\n\n    def get_tables(self) -> Response:\n        \"\"\"\n        List all files\n        \"\"\"\n        files_meta = self.file_controller.get_files()\n        data = [{\n            'TABLE_NAME': x['name'],\n            'TABLE_ROWS': x['row_count']\n        } for x in files_meta]\n        return Response(\n            RESPONSE_TYPE.TABLE,\n            data_frame=pd.DataFrame(data)\n        )\n\n    def get_columns(self, table_name) -> Response:\n        file_meta = self.file_controller.get_file_meta(table_name)\n        result = Response(\n            RESPONSE_TYPE.TABLE,\n            data_frame=pd.DataFrame([\n                {\n                    'Field': x.strip(),\n                    'Type': 'str'\n                } for x in file_meta['columns']\n            ])\n        )\n        return result\n", "path": "mindsdb/integrations/handlers/file_handler/file_handler.py"}], "after_files": [{"content": "from io import BytesIO, StringIO\nimport os\nimport csv\nimport json\nimport codecs\nimport traceback\nimport tempfile\nfrom urllib.parse import urlparse\n\nimport requests\nimport pandas as pd\nfrom charset_normalizer import from_bytes\n\nfrom mindsdb_sql import parse_sql\nfrom mindsdb_sql.parser.ast.base import ASTNode\nfrom mindsdb_sql.parser.ast import DropTables, Select\n\nfrom mindsdb.api.mysql.mysql_proxy.utilities.sql import query_df\nfrom mindsdb.integrations.libs.base_handler import DatabaseHandler\nfrom mindsdb.integrations.libs.response import (\n    HandlerStatusResponse as StatusResponse,\n    HandlerResponse as Response,\n    RESPONSE_TYPE\n)\n\n\ndef clean_row(row):\n    n_row = []\n    for cell in row:\n        if str(cell) in ['', ' ', '  ', 'NaN', 'nan', 'NA']:\n            n_row.append(None)\n        else:\n            n_row.append(cell)\n\n    return n_row\n\n\nclass FileHandler(DatabaseHandler):\n    \"\"\"\n    Handler for files\n    \"\"\"\n    name = 'files'\n\n    def __init__(self, name=None, file_storage=None, connection_data={}, file_controller=None):\n        super().__init__(name)\n        self.parser = parse_sql\n        self.fs_store = file_storage\n        self.custom_parser = connection_data.get('custom_parser', None)\n        self.clean_rows = connection_data.get('clean_rows', True)\n        self.file_controller = file_controller\n\n    def connect(self, **kwargs):\n        return\n\n    def disconnect(self, **kwargs):\n        return\n\n    def check_connection(self) -> StatusResponse:\n        return StatusResponse(True)\n\n    def query(self, query: ASTNode) -> Response:\n        if type(query) == DropTables:\n            for table_identifier in query.tables:\n                if len(table_identifier.parts) == 2 and table_identifier.parts[0] != self.name:\n                    return Response(\n                        RESPONSE_TYPE.ERROR,\n                        error_message=f\"Can't delete table from database '{table_identifier.parts[0]}'\"\n                    )\n                table_name = table_identifier.parts[-1]\n                try:\n                    self.file_controller.delete_file(table_name)\n                except Exception as e:\n                    return Response(\n                        RESPONSE_TYPE.ERROR,\n                        error_message=f\"Can't delete table '{table_name}': {e}\"\n                    )\n            return Response(RESPONSE_TYPE.OK)\n        elif type(query) == Select:\n            table_name = query.from_table.parts[-1]\n            file_path = self.file_controller.get_file_path(table_name)\n            df, _columns = self._handle_source(file_path, self.clean_rows, self.custom_parser)\n            result_df = query_df(df, query)\n            return Response(\n                RESPONSE_TYPE.TABLE,\n                data_frame=result_df\n            )\n        else:\n            return Response(\n                RESPONSE_TYPE.ERROR,\n                error_message=\"Only 'select' and 'drop' queries allowed for files\"\n            )\n\n    def native_query(self, query: str) -> Response:\n        ast = self.parser(query, dialect='mindsdb')\n        return self.query(ast)\n\n    @staticmethod\n    def _handle_source(file_path, clean_rows=True, custom_parser=None):\n        # get file data io, format and dialect\n        data, fmt, dialect = FileHandler._get_data_io(file_path)\n        data.seek(0)  # make sure we are at 0 in file pointer\n\n        if custom_parser:\n            header, file_data = custom_parser(data, fmt)\n\n        elif fmt == 'csv':\n            csv_reader = list(csv.reader(data, dialect))\n            header = csv_reader[0]\n            file_data = csv_reader[1:]\n\n        elif fmt in ['xlsx', 'xls']:\n            data.seek(0)\n            df = pd.read_excel(data)\n            header = df.columns.values.tolist()\n            file_data = df.values.tolist()\n\n        elif fmt == 'json':\n            data.seek(0)\n            json_doc = json.loads(data.read())\n            df = pd.json_normalize(json_doc, max_level=0)\n            header = df.columns.values.tolist()\n            file_data = df.values.tolist()\n\n        else:\n            raise ValueError('Could not load file into any format, supported formats are csv, json, xls, xlsx')\n\n        if clean_rows:\n            file_list_data = [clean_row(row) for row in file_data]\n        else:\n            file_list_data = file_data\n\n        header = [x.strip() for x in header]\n        col_map = dict((col, col) for col in header)\n        return pd.DataFrame(file_list_data, columns=header), col_map\n\n    @staticmethod\n    def _get_data_io(file_path):\n        \"\"\"\n        This gets a file either url or local file and defines what the format is as well as dialect\n        :param file: file path or url\n        :return: data_io, format, dialect\n        \"\"\"\n\n        ############\n        # get file as io object\n        ############\n\n        # file_path = self._get_file_path()\n\n        data = BytesIO()\n\n        try:\n            with open(file_path, 'rb') as fp:\n                data = BytesIO(fp.read())\n        except Exception as e:\n            error = 'Could not load file, possible exception : {exception}'.format(exception=e)\n            print(error)\n            raise ValueError(error)\n\n        dialect = None\n\n        ############\n        # check for file type\n        ############\n\n        # try to guess if its an excel file\n        xlsx_sig = b'\\x50\\x4B\\x05\\06'\n        # xlsx_sig2 = b'\\x50\\x4B\\x03\\x04'\n        xls_sig = b'\\x09\\x08\\x10\\x00\\x00\\x06\\x05\\x00'\n\n        # different whence, offset, size for different types\n        excel_meta = [('xls', 0, 512, 8), ('xlsx', 2, -22, 4)]\n\n        for filename, whence, offset, size in excel_meta:\n\n            try:\n                data.seek(offset, whence)  # Seek to the offset.\n                bytes = data.read(size)  # Capture the specified number of bytes.\n                data.seek(0)\n                codecs.getencoder('hex')(bytes)\n\n                if bytes == xls_sig:\n                    return data, 'xls', dialect\n                elif bytes == xlsx_sig:\n                    return data, 'xlsx', dialect\n\n            except Exception:\n                data.seek(0)\n\n        # if not excel it can be a json file or a CSV, convert from binary to stringio\n\n        byte_str = data.read()\n        # Move it to StringIO\n        try:\n            # Handle Microsoft's BOM \"special\" UTF-8 encoding\n            if byte_str.startswith(codecs.BOM_UTF8):\n                data = StringIO(byte_str.decode('utf-8-sig'))\n            else:\n                file_encoding_meta = from_bytes(\n                    byte_str[:32 * 1024],\n                    steps=32,           # Number of steps/block to extract from my_byte_str\n                    chunk_size=1024,    # Set block size of each extraction)\n                    explain=False\n                )\n                best_meta = file_encoding_meta.best()\n                errors = 'strict'\n                if best_meta is not None:\n                    encoding = file_encoding_meta.best().encoding\n                else:\n                    encoding = 'utf-8'\n                    errors = 'replace'\n                data = StringIO(byte_str.decode(encoding, errors))\n        except Exception:\n            print(traceback.format_exc())\n            print('Could not load into string')\n\n        # see if its JSON\n        buffer = data.read(100)\n        data.seek(0)\n        text = buffer.strip()\n        # analyze first n characters\n        if len(text) > 0:\n            text = text.strip()\n            # it it looks like a json, then try to parse it\n            if text.startswith('{') or text.startswith('['):\n                try:\n                    json.loads(data.read())\n                    data.seek(0)\n                    return data, 'json', dialect\n                except Exception:\n                    data.seek(0)\n                    return data, None, dialect\n\n        # lets try to figure out if its a csv\n        try:\n            dialect = FileHandler._get_csv_dialect(data)\n            if dialect:\n                return data, 'csv', dialect\n            return data, None, dialect\n        except Exception:\n            data.seek(0)\n            print('Could not detect format for this file')\n            print(traceback.format_exc())\n            # No file type identified\n            return data, None, dialect\n\n    @staticmethod\n    def _get_file_path(path) -> str:\n        try:\n            is_url = urlparse(path).scheme in ('http', 'https')\n        except Exception:\n            is_url = False\n        if is_url:\n            path = FileHandler._fetch_url(path)\n        return path\n\n    @staticmethod\n    def _get_csv_dialect(buffer) -> csv.Dialect:\n        sample = buffer.read(128 * 1024)\n        buffer.seek(0)\n        try:\n            accepted_csv_delimiters = [',', '\\t', ';']\n            dialect = csv.Sniffer().sniff(sample, delimiters=accepted_csv_delimiters)\n        except csv.Error:\n            dialect = None\n        return dialect\n\n    @staticmethod\n    def _fetch_url(url: str) -> str:\n        temp_dir = tempfile.mkdtemp(prefix='mindsdb_file_url_')\n        try:\n            r = requests.get(url, stream=True)\n            if r.status_code == 200:\n                with open(os.path.join(temp_dir, 'file'), 'wb') as f:\n                    for chunk in r:\n                        f.write(chunk)\n            else:\n                raise Exception(f'Responce status code is {r.status_code}')\n        except Exception as e:\n            print(f'Error during getting {url}')\n            print(e)\n            raise\n        return os.path.join(temp_dir, 'file')\n\n    def get_tables(self) -> Response:\n        \"\"\"\n        List all files\n        \"\"\"\n        files_meta = self.file_controller.get_files()\n        data = [{\n            'TABLE_NAME': x['name'],\n            'TABLE_ROWS': x['row_count']\n        } for x in files_meta]\n        return Response(\n            RESPONSE_TYPE.TABLE,\n            data_frame=pd.DataFrame(data)\n        )\n\n    def get_columns(self, table_name) -> Response:\n        file_meta = self.file_controller.get_file_meta(table_name)\n        result = Response(\n            RESPONSE_TYPE.TABLE,\n            data_frame=pd.DataFrame([\n                {\n                    'Field': x.strip(),\n                    'Type': 'str'\n                } for x in file_meta['columns']\n            ])\n        )\n        return result\n", "path": "mindsdb/integrations/handlers/file_handler/file_handler.py"}]}