We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Chunks number not correctly reported in statuscontainer
**Describe the bug**
Chunks number for non-PDF files not correctly reported in statuscontainer

**To Reproduce**
Steps to reproduce the behavior:
1. Upload a non-PDF file
2. Wait for the chunking process to complete
3. Go to CosmosDB -> statuscontainer -> item relative to the uploaded non-PDF file
4. Check the "status" field which will display something like: 
"FileLayoutParsingOther - chunking complete. <built-in method count of list object at 0x7713c4227440> chunks created" 

**Expected behavior**
It should show a number instead of "<built-in method count of list object at 0x7713c4227440>"

**Desktop (please complete the following information):**
 - OS: Windows 11
 - Browser: Edge
 - Version: 120.0.2210.121 (Official build) (64-bit)

**Alpha version details**
 - GitHub branch: main

**Additional context**
Minor problem
probably line 166, functions/FileLayoutParsingOther/__init__.py


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `functions/FileLayoutParsingOther/__init__.py`
Content:
```
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT license.
3 
4 import logging
5 import os
6 import json
7 from enum import Enum
8 from io import BytesIO
9 import azure.functions as func
10 from azure.storage.blob import generate_blob_sas
11 from azure.storage.queue import QueueClient, TextBase64EncodePolicy
12 from shared_code.status_log import StatusLog, State, StatusClassification
13 from shared_code.utilities import Utilities, MediaType
14 
15 import requests
16 
17 azure_blob_storage_account = os.environ["BLOB_STORAGE_ACCOUNT"]
18 azure_blob_storage_endpoint = os.environ["BLOB_STORAGE_ACCOUNT_ENDPOINT"]
19 azure_blob_drop_storage_container = os.environ["BLOB_STORAGE_ACCOUNT_UPLOAD_CONTAINER_NAME"]
20 azure_blob_content_storage_container = os.environ["BLOB_STORAGE_ACCOUNT_OUTPUT_CONTAINER_NAME"]
21 azure_blob_storage_key = os.environ["AZURE_BLOB_STORAGE_KEY"]
22 azure_blob_connection_string = os.environ["BLOB_CONNECTION_STRING"]
23 azure_blob_log_storage_container = os.environ["BLOB_STORAGE_ACCOUNT_LOG_CONTAINER_NAME"]
24 cosmosdb_url = os.environ["COSMOSDB_URL"]
25 cosmosdb_key = os.environ["COSMOSDB_KEY"]
26 cosmosdb_log_database_name = os.environ["COSMOSDB_LOG_DATABASE_NAME"]
27 cosmosdb_log_container_name = os.environ["COSMOSDB_LOG_CONTAINER_NAME"]
28 non_pdf_submit_queue = os.environ["NON_PDF_SUBMIT_QUEUE"]
29 pdf_polling_queue = os.environ["PDF_POLLING_QUEUE"]
30 pdf_submit_queue = os.environ["PDF_SUBMIT_QUEUE"]
31 text_enrichment_queue = os.environ["TEXT_ENRICHMENT_QUEUE"]
32 CHUNK_TARGET_SIZE = int(os.environ["CHUNK_TARGET_SIZE"])
33 
34 NEW_AFTER_N_CHARS = 1500
35 COMBINE_UNDER_N_CHARS = 500
36 MAX_CHARACTERS = 1500
37 
38 
39 utilities = Utilities(azure_blob_storage_account, azure_blob_storage_endpoint, azure_blob_drop_storage_container, azure_blob_content_storage_container, azure_blob_storage_key)
40 function_name = "FileLayoutParsingOther"
41 
42 class UnstructuredError(Exception):
43     pass
44 
45 def PartitionFile(file_extension: str, file_url: str):      
46     """ uses the unstructured.io libraries to analyse a document
47     Returns:
48         elements: A list of available models
49     """  
50     # Send a GET request to the URL to download the file
51     response = requests.get(file_url)
52     bytes_io = BytesIO(response.content)
53     response.close()   
54     metadata = [] 
55     elements = None
56     try:        
57         if file_extension == '.csv':
58             from unstructured.partition.csv import partition_csv
59             elements = partition_csv(file=bytes_io)               
60                      
61         elif file_extension == '.doc':
62             from unstructured.partition.doc import partition_doc
63             elements = partition_doc(file=bytes_io) 
64             
65         elif file_extension == '.docx':
66             from unstructured.partition.docx import partition_docx
67             elements = partition_docx(file=bytes_io)
68             
69         elif file_extension == '.eml' or file_extension == '.msg':
70             if file_extension == '.msg':
71                 from unstructured.partition.msg import partition_msg
72                 elements = partition_msg(file=bytes_io) 
73             else:        
74                 from unstructured.partition.email import partition_email
75                 elements = partition_email(file=bytes_io)
76             metadata.append(f'Subject: {elements[0].metadata.subject}')
77             metadata.append(f'From: {elements[0].metadata.sent_from[0]}')
78             sent_to_str = 'To: '
79             for sent_to in elements[0].metadata.sent_to:
80                 sent_to_str = sent_to_str + " " + sent_to
81             metadata.append(sent_to_str)
82             
83         elif file_extension == '.html' or file_extension == '.htm':  
84             from unstructured.partition.html import partition_html
85             elements = partition_html(file=bytes_io) 
86             
87         elif file_extension == '.md':
88             from unstructured.partition.md import partition_md
89             elements = partition_md(file=bytes_io)
90                        
91         elif file_extension == '.ppt':
92             from unstructured.partition.ppt import partition_ppt
93             elements = partition_ppt(file=bytes_io)
94             
95         elif file_extension == '.pptx':    
96             from unstructured.partition.pptx import partition_pptx
97             elements = partition_pptx(file=bytes_io)
98             
99         elif any(file_extension in x for x in ['.txt', '.json']):
100             from unstructured.partition.text import partition_text
101             elements = partition_text(file=bytes_io)
102             
103         elif file_extension == '.xlsx':
104             from unstructured.partition.xlsx import partition_xlsx
105             elements = partition_xlsx(file=bytes_io)
106             
107         elif file_extension == '.xml':
108             from unstructured.partition.xml import partition_xml
109             elements = partition_xml(file=bytes_io)
110             
111     except Exception as e:
112         raise UnstructuredError(f"An error occurred trying to parse the file: {str(e)}") from e
113          
114     return elements, metadata
115     
116     
117 
118 def main(msg: func.QueueMessage) -> None:
119     try:
120         statusLog = StatusLog(cosmosdb_url, cosmosdb_key, cosmosdb_log_database_name, cosmosdb_log_container_name)
121         logging.info('Python queue trigger function processed a queue item: %s',
122                     msg.get_body().decode('utf-8'))
123 
124         # Receive message from the queue
125         message_body = msg.get_body().decode('utf-8')
126         message_json = json.loads(message_body)
127         blob_name =  message_json['blob_name']
128         blob_uri =  message_json['blob_uri']
129         statusLog.upsert_document(blob_name, f'{function_name} - Starting to parse the non-PDF file', StatusClassification.INFO, State.PROCESSING)
130         statusLog.upsert_document(blob_name, f'{function_name} - Message received from non-pdf submit queue', StatusClassification.DEBUG)
131 
132         # construct blob url
133         blob_path_plus_sas = utilities.get_blob_and_sas(blob_name)
134         statusLog.upsert_document(blob_name, f'{function_name} - SAS token generated to access the file', StatusClassification.DEBUG)
135 
136         file_name, file_extension, file_directory  = utilities.get_filename_and_extension(blob_name)
137 
138         response = requests.get(blob_path_plus_sas)
139         response.raise_for_status()
140               
141         
142         # Partition the file dependent on file extension
143         elements, metadata = PartitionFile(file_extension, blob_path_plus_sas)
144         metdata_text = ''
145         for metadata_value in metadata:
146             metdata_text += metadata_value + '\n'    
147         statusLog.upsert_document(blob_name, f'{function_name} - partitioning complete', StatusClassification.DEBUG)
148         
149         title = ''
150         # Capture the file title
151         try:
152             for i, element in enumerate(elements):
153                 if title == '' and element.category == 'Title':
154                     # capture the first title
155                     title = element.text
156                     break
157         except:
158             # if this type of eleemnt does not include title, then process with emty value
159             pass
160         
161         # Chunk the file     
162         from unstructured.chunking.title import chunk_by_title
163         # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS)
164         # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS, max_characters=MAX_CHARACTERS)   
165         chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_text_under_n_chars=COMBINE_UNDER_N_CHARS)
166         statusLog.upsert_document(blob_name, f'{function_name} - chunking complete. {str(chunks.count)} chunks created', StatusClassification.DEBUG)
167                 
168         subtitle_name = ''
169         section_name = ''
170         # Complete and write chunks
171         for i, chunk in enumerate(chunks):      
172             if chunk.metadata.page_number == None:
173                 page_list = [1]
174             else:
175                 page_list = [chunk.metadata.page_number] 
176             # substitute html if text is a table            
177             if chunk.category == 'Table':
178                 chunk_text = chunk.metadata.text_as_html
179             else:
180                 chunk_text = chunk.text
181             # add filetype specific metadata as chunk text header
182             chunk_text = metdata_text + chunk_text                    
183             utilities.write_chunk(blob_name, blob_uri,
184                                 f"{i}",
185                                 utilities.token_count(chunk.text),
186                                 chunk_text, page_list,
187                                 section_name, title, subtitle_name,
188                                 MediaType.TEXT
189                                 )
190         
191         statusLog.upsert_document(blob_name, f'{function_name} - chunking stored.', StatusClassification.DEBUG)   
192         
193         # submit message to the text enrichment queue to continue processing                
194         queue_client = QueueClient.from_connection_string(azure_blob_connection_string, queue_name=text_enrichment_queue, message_encode_policy=TextBase64EncodePolicy())
195         message_json["text_enrichment_queued_count"] = 1
196         message_string = json.dumps(message_json)
197         queue_client.send_message(message_string)
198         statusLog.upsert_document(blob_name, f"{function_name} - message sent to enrichment queue", StatusClassification.DEBUG, State.QUEUED)    
199              
200     except Exception as e:
201         statusLog.upsert_document(blob_name, f"{function_name} - An error occurred - {str(e)}", StatusClassification.ERROR, State.ERROR)
202 
203     statusLog.save_document(blob_name)
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/functions/FileLayoutParsingOther/__init__.py b/functions/FileLayoutParsingOther/__init__.py
--- a/functions/FileLayoutParsingOther/__init__.py
+++ b/functions/FileLayoutParsingOther/__init__.py
@@ -163,7 +163,7 @@
         # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS)
         # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS, max_characters=MAX_CHARACTERS)   
         chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_text_under_n_chars=COMBINE_UNDER_N_CHARS)
-        statusLog.upsert_document(blob_name, f'{function_name} - chunking complete. {str(chunks.count)} chunks created', StatusClassification.DEBUG)
+        statusLog.upsert_document(blob_name, f'{function_name} - chunking complete. {len(chunks)} chunks created', StatusClassification.DEBUG)
                 
         subtitle_name = ''
         section_name = ''

{"golden_diff": "diff --git a/functions/FileLayoutParsingOther/__init__.py b/functions/FileLayoutParsingOther/__init__.py\n--- a/functions/FileLayoutParsingOther/__init__.py\n+++ b/functions/FileLayoutParsingOther/__init__.py\n@@ -163,7 +163,7 @@\n         # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS)\r\n         # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS, max_characters=MAX_CHARACTERS)   \r\n         chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_text_under_n_chars=COMBINE_UNDER_N_CHARS)\r\n-        statusLog.upsert_document(blob_name, f'{function_name} - chunking complete. {str(chunks.count)} chunks created', StatusClassification.DEBUG)\r\n+        statusLog.upsert_document(blob_name, f'{function_name} - chunking complete. {len(chunks)} chunks created', StatusClassification.DEBUG)\r\n                 \r\n         subtitle_name = ''\r\n         section_name = ''\n", "issue": "Chunks number not correctly reported in statuscontainer\n**Describe the bug**\r\nChunks number for non-PDF files not correctly reported in statuscontainer\r\n\r\n**To Reproduce**\r\nSteps to reproduce the behavior:\r\n1. Upload a non-PDF file\r\n2. Wait for the chunking process to complete\r\n3. Go to CosmosDB -> statuscontainer -> item relative to the uploaded non-PDF file\r\n4. Check the \"status\" field which will display something like: \r\n\"FileLayoutParsingOther - chunking complete. <built-in method count of list object at 0x7713c4227440> chunks created\" \r\n\r\n**Expected behavior**\r\nIt should show a number instead of \"<built-in method count of list object at 0x7713c4227440>\"\r\n\r\n**Desktop (please complete the following information):**\r\n - OS: Windows 11\r\n - Browser: Edge\r\n - Version: 120.0.2210.121 (Official build) (64-bit)\r\n\r\n**Alpha version details**\r\n - GitHub branch: main\r\n\r\n**Additional context**\r\nMinor problem\r\nprobably line 166, functions/FileLayoutParsingOther/__init__.py\r\n\n", "before_files": [{"content": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT license.\r\n\r\nimport logging\r\nimport os\r\nimport json\r\nfrom enum import Enum\r\nfrom io import BytesIO\r\nimport azure.functions as func\r\nfrom azure.storage.blob import generate_blob_sas\r\nfrom azure.storage.queue import QueueClient, TextBase64EncodePolicy\r\nfrom shared_code.status_log import StatusLog, State, StatusClassification\r\nfrom shared_code.utilities import Utilities, MediaType\r\n\r\nimport requests\r\n\r\nazure_blob_storage_account = os.environ[\"BLOB_STORAGE_ACCOUNT\"]\r\nazure_blob_storage_endpoint = os.environ[\"BLOB_STORAGE_ACCOUNT_ENDPOINT\"]\r\nazure_blob_drop_storage_container = os.environ[\"BLOB_STORAGE_ACCOUNT_UPLOAD_CONTAINER_NAME\"]\r\nazure_blob_content_storage_container = os.environ[\"BLOB_STORAGE_ACCOUNT_OUTPUT_CONTAINER_NAME\"]\r\nazure_blob_storage_key = os.environ[\"AZURE_BLOB_STORAGE_KEY\"]\r\nazure_blob_connection_string = os.environ[\"BLOB_CONNECTION_STRING\"]\r\nazure_blob_log_storage_container = os.environ[\"BLOB_STORAGE_ACCOUNT_LOG_CONTAINER_NAME\"]\r\ncosmosdb_url = os.environ[\"COSMOSDB_URL\"]\r\ncosmosdb_key = os.environ[\"COSMOSDB_KEY\"]\r\ncosmosdb_log_database_name = os.environ[\"COSMOSDB_LOG_DATABASE_NAME\"]\r\ncosmosdb_log_container_name = os.environ[\"COSMOSDB_LOG_CONTAINER_NAME\"]\r\nnon_pdf_submit_queue = os.environ[\"NON_PDF_SUBMIT_QUEUE\"]\r\npdf_polling_queue = os.environ[\"PDF_POLLING_QUEUE\"]\r\npdf_submit_queue = os.environ[\"PDF_SUBMIT_QUEUE\"]\r\ntext_enrichment_queue = os.environ[\"TEXT_ENRICHMENT_QUEUE\"]\r\nCHUNK_TARGET_SIZE = int(os.environ[\"CHUNK_TARGET_SIZE\"])\r\n\r\nNEW_AFTER_N_CHARS = 1500\r\nCOMBINE_UNDER_N_CHARS = 500\r\nMAX_CHARACTERS = 1500\r\n\r\n\r\nutilities = Utilities(azure_blob_storage_account, azure_blob_storage_endpoint, azure_blob_drop_storage_container, azure_blob_content_storage_container, azure_blob_storage_key)\r\nfunction_name = \"FileLayoutParsingOther\"\r\n\r\nclass UnstructuredError(Exception):\r\n    pass\r\n\r\ndef PartitionFile(file_extension: str, file_url: str):      \r\n    \"\"\" uses the unstructured.io libraries to analyse a document\r\n    Returns:\r\n        elements: A list of available models\r\n    \"\"\"  \r\n    # Send a GET request to the URL to download the file\r\n    response = requests.get(file_url)\r\n    bytes_io = BytesIO(response.content)\r\n    response.close()   \r\n    metadata = [] \r\n    elements = None\r\n    try:        \r\n        if file_extension == '.csv':\r\n            from unstructured.partition.csv import partition_csv\r\n            elements = partition_csv(file=bytes_io)               \r\n                     \r\n        elif file_extension == '.doc':\r\n            from unstructured.partition.doc import partition_doc\r\n            elements = partition_doc(file=bytes_io) \r\n            \r\n        elif file_extension == '.docx':\r\n            from unstructured.partition.docx import partition_docx\r\n            elements = partition_docx(file=bytes_io)\r\n            \r\n        elif file_extension == '.eml' or file_extension == '.msg':\r\n            if file_extension == '.msg':\r\n                from unstructured.partition.msg import partition_msg\r\n                elements = partition_msg(file=bytes_io) \r\n            else:        \r\n                from unstructured.partition.email import partition_email\r\n                elements = partition_email(file=bytes_io)\r\n            metadata.append(f'Subject: {elements[0].metadata.subject}')\r\n            metadata.append(f'From: {elements[0].metadata.sent_from[0]}')\r\n            sent_to_str = 'To: '\r\n            for sent_to in elements[0].metadata.sent_to:\r\n                sent_to_str = sent_to_str + \" \" + sent_to\r\n            metadata.append(sent_to_str)\r\n            \r\n        elif file_extension == '.html' or file_extension == '.htm':  \r\n            from unstructured.partition.html import partition_html\r\n            elements = partition_html(file=bytes_io) \r\n            \r\n        elif file_extension == '.md':\r\n            from unstructured.partition.md import partition_md\r\n            elements = partition_md(file=bytes_io)\r\n                       \r\n        elif file_extension == '.ppt':\r\n            from unstructured.partition.ppt import partition_ppt\r\n            elements = partition_ppt(file=bytes_io)\r\n            \r\n        elif file_extension == '.pptx':    \r\n            from unstructured.partition.pptx import partition_pptx\r\n            elements = partition_pptx(file=bytes_io)\r\n            \r\n        elif any(file_extension in x for x in ['.txt', '.json']):\r\n            from unstructured.partition.text import partition_text\r\n            elements = partition_text(file=bytes_io)\r\n            \r\n        elif file_extension == '.xlsx':\r\n            from unstructured.partition.xlsx import partition_xlsx\r\n            elements = partition_xlsx(file=bytes_io)\r\n            \r\n        elif file_extension == '.xml':\r\n            from unstructured.partition.xml import partition_xml\r\n            elements = partition_xml(file=bytes_io)\r\n            \r\n    except Exception as e:\r\n        raise UnstructuredError(f\"An error occurred trying to parse the file: {str(e)}\") from e\r\n         \r\n    return elements, metadata\r\n    \r\n    \r\n\r\ndef main(msg: func.QueueMessage) -> None:\r\n    try:\r\n        statusLog = StatusLog(cosmosdb_url, cosmosdb_key, cosmosdb_log_database_name, cosmosdb_log_container_name)\r\n        logging.info('Python queue trigger function processed a queue item: %s',\r\n                    msg.get_body().decode('utf-8'))\r\n\r\n        # Receive message from the queue\r\n        message_body = msg.get_body().decode('utf-8')\r\n        message_json = json.loads(message_body)\r\n        blob_name =  message_json['blob_name']\r\n        blob_uri =  message_json['blob_uri']\r\n        statusLog.upsert_document(blob_name, f'{function_name} - Starting to parse the non-PDF file', StatusClassification.INFO, State.PROCESSING)\r\n        statusLog.upsert_document(blob_name, f'{function_name} - Message received from non-pdf submit queue', StatusClassification.DEBUG)\r\n\r\n        # construct blob url\r\n        blob_path_plus_sas = utilities.get_blob_and_sas(blob_name)\r\n        statusLog.upsert_document(blob_name, f'{function_name} - SAS token generated to access the file', StatusClassification.DEBUG)\r\n\r\n        file_name, file_extension, file_directory  = utilities.get_filename_and_extension(blob_name)\r\n\r\n        response = requests.get(blob_path_plus_sas)\r\n        response.raise_for_status()\r\n              \r\n        \r\n        # Partition the file dependent on file extension\r\n        elements, metadata = PartitionFile(file_extension, blob_path_plus_sas)\r\n        metdata_text = ''\r\n        for metadata_value in metadata:\r\n            metdata_text += metadata_value + '\\n'    \r\n        statusLog.upsert_document(blob_name, f'{function_name} - partitioning complete', StatusClassification.DEBUG)\r\n        \r\n        title = ''\r\n        # Capture the file title\r\n        try:\r\n            for i, element in enumerate(elements):\r\n                if title == '' and element.category == 'Title':\r\n                    # capture the first title\r\n                    title = element.text\r\n                    break\r\n        except:\r\n            # if this type of eleemnt does not include title, then process with emty value\r\n            pass\r\n        \r\n        # Chunk the file     \r\n        from unstructured.chunking.title import chunk_by_title\r\n        # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS)\r\n        # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS, max_characters=MAX_CHARACTERS)   \r\n        chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_text_under_n_chars=COMBINE_UNDER_N_CHARS)\r\n        statusLog.upsert_document(blob_name, f'{function_name} - chunking complete. {str(chunks.count)} chunks created', StatusClassification.DEBUG)\r\n                \r\n        subtitle_name = ''\r\n        section_name = ''\r\n        # Complete and write chunks\r\n        for i, chunk in enumerate(chunks):      \r\n            if chunk.metadata.page_number == None:\r\n                page_list = [1]\r\n            else:\r\n                page_list = [chunk.metadata.page_number] \r\n            # substitute html if text is a table            \r\n            if chunk.category == 'Table':\r\n                chunk_text = chunk.metadata.text_as_html\r\n            else:\r\n                chunk_text = chunk.text\r\n            # add filetype specific metadata as chunk text header\r\n            chunk_text = metdata_text + chunk_text                    \r\n            utilities.write_chunk(blob_name, blob_uri,\r\n                                f\"{i}\",\r\n                                utilities.token_count(chunk.text),\r\n                                chunk_text, page_list,\r\n                                section_name, title, subtitle_name,\r\n                                MediaType.TEXT\r\n                                )\r\n        \r\n        statusLog.upsert_document(blob_name, f'{function_name} - chunking stored.', StatusClassification.DEBUG)   \r\n        \r\n        # submit message to the text enrichment queue to continue processing                \r\n        queue_client = QueueClient.from_connection_string(azure_blob_connection_string, queue_name=text_enrichment_queue, message_encode_policy=TextBase64EncodePolicy())\r\n        message_json[\"text_enrichment_queued_count\"] = 1\r\n        message_string = json.dumps(message_json)\r\n        queue_client.send_message(message_string)\r\n        statusLog.upsert_document(blob_name, f\"{function_name} - message sent to enrichment queue\", StatusClassification.DEBUG, State.QUEUED)    \r\n             \r\n    except Exception as e:\r\n        statusLog.upsert_document(blob_name, f\"{function_name} - An error occurred - {str(e)}\", StatusClassification.ERROR, State.ERROR)\r\n\r\n    statusLog.save_document(blob_name)", "path": "functions/FileLayoutParsingOther/__init__.py"}], "after_files": [{"content": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT license.\r\n\r\nimport logging\r\nimport os\r\nimport json\r\nfrom enum import Enum\r\nfrom io import BytesIO\r\nimport azure.functions as func\r\nfrom azure.storage.blob import generate_blob_sas\r\nfrom azure.storage.queue import QueueClient, TextBase64EncodePolicy\r\nfrom shared_code.status_log import StatusLog, State, StatusClassification\r\nfrom shared_code.utilities import Utilities, MediaType\r\n\r\nimport requests\r\n\r\nazure_blob_storage_account = os.environ[\"BLOB_STORAGE_ACCOUNT\"]\r\nazure_blob_storage_endpoint = os.environ[\"BLOB_STORAGE_ACCOUNT_ENDPOINT\"]\r\nazure_blob_drop_storage_container = os.environ[\"BLOB_STORAGE_ACCOUNT_UPLOAD_CONTAINER_NAME\"]\r\nazure_blob_content_storage_container = os.environ[\"BLOB_STORAGE_ACCOUNT_OUTPUT_CONTAINER_NAME\"]\r\nazure_blob_storage_key = os.environ[\"AZURE_BLOB_STORAGE_KEY\"]\r\nazure_blob_connection_string = os.environ[\"BLOB_CONNECTION_STRING\"]\r\nazure_blob_log_storage_container = os.environ[\"BLOB_STORAGE_ACCOUNT_LOG_CONTAINER_NAME\"]\r\ncosmosdb_url = os.environ[\"COSMOSDB_URL\"]\r\ncosmosdb_key = os.environ[\"COSMOSDB_KEY\"]\r\ncosmosdb_log_database_name = os.environ[\"COSMOSDB_LOG_DATABASE_NAME\"]\r\ncosmosdb_log_container_name = os.environ[\"COSMOSDB_LOG_CONTAINER_NAME\"]\r\nnon_pdf_submit_queue = os.environ[\"NON_PDF_SUBMIT_QUEUE\"]\r\npdf_polling_queue = os.environ[\"PDF_POLLING_QUEUE\"]\r\npdf_submit_queue = os.environ[\"PDF_SUBMIT_QUEUE\"]\r\ntext_enrichment_queue = os.environ[\"TEXT_ENRICHMENT_QUEUE\"]\r\nCHUNK_TARGET_SIZE = int(os.environ[\"CHUNK_TARGET_SIZE\"])\r\n\r\nNEW_AFTER_N_CHARS = 1500\r\nCOMBINE_UNDER_N_CHARS = 500\r\nMAX_CHARACTERS = 1500\r\n\r\n\r\nutilities = Utilities(azure_blob_storage_account, azure_blob_storage_endpoint, azure_blob_drop_storage_container, azure_blob_content_storage_container, azure_blob_storage_key)\r\nfunction_name = \"FileLayoutParsingOther\"\r\n\r\nclass UnstructuredError(Exception):\r\n    pass\r\n\r\ndef PartitionFile(file_extension: str, file_url: str):      \r\n    \"\"\" uses the unstructured.io libraries to analyse a document\r\n    Returns:\r\n        elements: A list of available models\r\n    \"\"\"  \r\n    # Send a GET request to the URL to download the file\r\n    response = requests.get(file_url)\r\n    bytes_io = BytesIO(response.content)\r\n    response.close()   \r\n    metadata = [] \r\n    elements = None\r\n    try:        \r\n        if file_extension == '.csv':\r\n            from unstructured.partition.csv import partition_csv\r\n            elements = partition_csv(file=bytes_io)               \r\n                     \r\n        elif file_extension == '.doc':\r\n            from unstructured.partition.doc import partition_doc\r\n            elements = partition_doc(file=bytes_io) \r\n            \r\n        elif file_extension == '.docx':\r\n            from unstructured.partition.docx import partition_docx\r\n            elements = partition_docx(file=bytes_io)\r\n            \r\n        elif file_extension == '.eml' or file_extension == '.msg':\r\n            if file_extension == '.msg':\r\n                from unstructured.partition.msg import partition_msg\r\n                elements = partition_msg(file=bytes_io) \r\n            else:        \r\n                from unstructured.partition.email import partition_email\r\n                elements = partition_email(file=bytes_io)\r\n            metadata.append(f'Subject: {elements[0].metadata.subject}')\r\n            metadata.append(f'From: {elements[0].metadata.sent_from[0]}')\r\n            sent_to_str = 'To: '\r\n            for sent_to in elements[0].metadata.sent_to:\r\n                sent_to_str = sent_to_str + \" \" + sent_to\r\n            metadata.append(sent_to_str)\r\n            \r\n        elif file_extension == '.html' or file_extension == '.htm':  \r\n            from unstructured.partition.html import partition_html\r\n            elements = partition_html(file=bytes_io) \r\n            \r\n        elif file_extension == '.md':\r\n            from unstructured.partition.md import partition_md\r\n            elements = partition_md(file=bytes_io)\r\n                       \r\n        elif file_extension == '.ppt':\r\n            from unstructured.partition.ppt import partition_ppt\r\n            elements = partition_ppt(file=bytes_io)\r\n            \r\n        elif file_extension == '.pptx':    \r\n            from unstructured.partition.pptx import partition_pptx\r\n            elements = partition_pptx(file=bytes_io)\r\n            \r\n        elif any(file_extension in x for x in ['.txt', '.json']):\r\n            from unstructured.partition.text import partition_text\r\n            elements = partition_text(file=bytes_io)\r\n            \r\n        elif file_extension == '.xlsx':\r\n            from unstructured.partition.xlsx import partition_xlsx\r\n            elements = partition_xlsx(file=bytes_io)\r\n            \r\n        elif file_extension == '.xml':\r\n            from unstructured.partition.xml import partition_xml\r\n            elements = partition_xml(file=bytes_io)\r\n            \r\n    except Exception as e:\r\n        raise UnstructuredError(f\"An error occurred trying to parse the file: {str(e)}\") from e\r\n         \r\n    return elements, metadata\r\n    \r\n    \r\n\r\ndef main(msg: func.QueueMessage) -> None:\r\n    try:\r\n        statusLog = StatusLog(cosmosdb_url, cosmosdb_key, cosmosdb_log_database_name, cosmosdb_log_container_name)\r\n        logging.info('Python queue trigger function processed a queue item: %s',\r\n                    msg.get_body().decode('utf-8'))\r\n\r\n        # Receive message from the queue\r\n        message_body = msg.get_body().decode('utf-8')\r\n        message_json = json.loads(message_body)\r\n        blob_name =  message_json['blob_name']\r\n        blob_uri =  message_json['blob_uri']\r\n        statusLog.upsert_document(blob_name, f'{function_name} - Starting to parse the non-PDF file', StatusClassification.INFO, State.PROCESSING)\r\n        statusLog.upsert_document(blob_name, f'{function_name} - Message received from non-pdf submit queue', StatusClassification.DEBUG)\r\n\r\n        # construct blob url\r\n        blob_path_plus_sas = utilities.get_blob_and_sas(blob_name)\r\n        statusLog.upsert_document(blob_name, f'{function_name} - SAS token generated to access the file', StatusClassification.DEBUG)\r\n\r\n        file_name, file_extension, file_directory  = utilities.get_filename_and_extension(blob_name)\r\n\r\n        response = requests.get(blob_path_plus_sas)\r\n        response.raise_for_status()\r\n              \r\n        \r\n        # Partition the file dependent on file extension\r\n        elements, metadata = PartitionFile(file_extension, blob_path_plus_sas)\r\n        metdata_text = ''\r\n        for metadata_value in metadata:\r\n            metdata_text += metadata_value + '\\n'    \r\n        statusLog.upsert_document(blob_name, f'{function_name} - partitioning complete', StatusClassification.DEBUG)\r\n        \r\n        title = ''\r\n        # Capture the file title\r\n        try:\r\n            for i, element in enumerate(elements):\r\n                if title == '' and element.category == 'Title':\r\n                    # capture the first title\r\n                    title = element.text\r\n                    break\r\n        except:\r\n            # if this type of eleemnt does not include title, then process with emty value\r\n            pass\r\n        \r\n        # Chunk the file     \r\n        from unstructured.chunking.title import chunk_by_title\r\n        # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS)\r\n        # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS, max_characters=MAX_CHARACTERS)   \r\n        chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_text_under_n_chars=COMBINE_UNDER_N_CHARS)\r\n        statusLog.upsert_document(blob_name, f'{function_name} - chunking complete. {len(chunks)} chunks created', StatusClassification.DEBUG)\r\n                \r\n        subtitle_name = ''\r\n        section_name = ''\r\n        # Complete and write chunks\r\n        for i, chunk in enumerate(chunks):      \r\n            if chunk.metadata.page_number == None:\r\n                page_list = [1]\r\n            else:\r\n                page_list = [chunk.metadata.page_number] \r\n            # substitute html if text is a table            \r\n            if chunk.category == 'Table':\r\n                chunk_text = chunk.metadata.text_as_html\r\n            else:\r\n                chunk_text = chunk.text\r\n            # add filetype specific metadata as chunk text header\r\n            chunk_text = metdata_text + chunk_text                    \r\n            utilities.write_chunk(blob_name, blob_uri,\r\n                                f\"{i}\",\r\n                                utilities.token_count(chunk.text),\r\n                                chunk_text, page_list,\r\n                                section_name, title, subtitle_name,\r\n                                MediaType.TEXT\r\n                                )\r\n        \r\n        statusLog.upsert_document(blob_name, f'{function_name} - chunking stored.', StatusClassification.DEBUG)   \r\n        \r\n        # submit message to the text enrichment queue to continue processing                \r\n        queue_client = QueueClient.from_connection_string(azure_blob_connection_string, queue_name=text_enrichment_queue, message_encode_policy=TextBase64EncodePolicy())\r\n        message_json[\"text_enrichment_queued_count\"] = 1\r\n        message_string = json.dumps(message_json)\r\n        queue_client.send_message(message_string)\r\n        statusLog.upsert_document(blob_name, f\"{function_name} - message sent to enrichment queue\", StatusClassification.DEBUG, State.QUEUED)    \r\n             \r\n    except Exception as e:\r\n        statusLog.upsert_document(blob_name, f\"{function_name} - An error occurred - {str(e)}\", StatusClassification.ERROR, State.ERROR)\r\n\r\n    statusLog.save_document(blob_name)", "path": "functions/FileLayoutParsingOther/__init__.py"}]}