We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[import-report] Trouble parsing domains / URLs with capital letters
## Description

import-report appears to fail when parsing domains with capital letters. I've attached the txt file of domains I was trying to import below.  The import fails with the error "'list' object has no attribute 'values'". I played with the input file by removing domains that appeared to be causing issues, and the problematic domains were:

- kAty197.chickenkiller[.]com
- Soure7788.chickenkiller[.]com
- Engaction[.]com

## Environment

1. OS (where OpenCTI server runs): Ubuntu 18
2. OpenCTI version: OpenCTI 4.5.5
3. OpenCTI client: frontend
4. Other environment details: import-connector: rolling

## Reproducible Steps

Steps to create the smallest reproducible scenario:
1. Import txt file with domains/URLs with capital letters

## Expected Output

Domains / URLs are parsed as observables

## Actual Output

Import-report connector crashes, no observables are parsed unless the capital letters become lowercase or the offending domains are removed
 
## Additional information

I've attached 3 files:

- **AA21-200A_domains_failed.txt** : The original txt file, with domains including capital letters
- **AA21-200A_domains_success.txt** : The revised txt file, with no capital letters
- **errorlog.txt** : The error log, from one of the times when I tried to import a version of the original file (this was an iteration of troubleshooting, so the error log shows what happens when I input the txt file having removed the first two offending domains)

[AA21-200A_domains_success.txt](https://github.com/OpenCTI-Platform/connectors/files/7101092/AA21-200A_domains_success.txt)
[errorlog.txt](https://github.com/OpenCTI-Platform/connectors/files/7101093/errorlog.txt)
[AA21-200A_domains_failed.txt](https://github.com/OpenCTI-Platform/connectors/files/7101094/AA21-200A_domains_failed.txt)



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `internal-import-file/import-report/src/reportimporter/report_parser.py`
Content:
```
1 import logging
2 import os
3 from typing import Dict, List, Pattern, IO, Tuple
4 
5 import ioc_finder
6 from pdfminer.high_level import extract_pages
7 from pdfminer.layout import LTTextContainer
8 from pycti import OpenCTIConnectorHelper
9 from reportimporter.constants import (
10     OBSERVABLE_CLASS,
11     ENTITY_CLASS,
12     RESULT_FORMAT_MATCH,
13     RESULT_FORMAT_TYPE,
14     RESULT_FORMAT_CATEGORY,
15     RESULT_FORMAT_RANGE,
16     MIME_PDF,
17     MIME_TXT,
18     OBSERVABLE_DETECTION_CUSTOM_REGEX,
19     OBSERVABLE_DETECTION_LIBRARY,
20 )
21 from reportimporter.models import Observable, Entity
22 from reportimporter.util import library_mapping
23 
24 
25 class ReportParser(object):
26     """
27     Report parser based on IOCParser
28     """
29 
30     def __init__(
31         self,
32         helper: OpenCTIConnectorHelper,
33         entity_list: List[Entity],
34         observable_list: List[Observable],
35     ):
36 
37         self.helper = helper
38         self.entity_list = entity_list
39         self.observable_list = observable_list
40 
41         # Disable INFO logging by pdfminer
42         logging.getLogger("pdfminer").setLevel(logging.WARNING)
43 
44         # Supported file types
45         self.supported_file_types = {
46             MIME_PDF: self._parse_pdf,
47             MIME_TXT: self._parse_text,
48         }
49 
50         self.library_lookup = library_mapping()
51 
52     def _is_whitelisted(self, regex_list: List[Pattern], ind_match: str):
53         for regex in regex_list:
54             self.helper.log_debug(f"Filter regex '{regex}' for value '{ind_match}'")
55             result = regex.search(ind_match)
56             if result:
57                 self.helper.log_debug(f"Value {ind_match} is whitelisted with {regex}")
58                 return True
59         return False
60 
61     def _post_parse_observables(
62         self, ind_match: str, observable: Observable, match_range: Tuple
63     ) -> Dict:
64         self.helper.log_debug(f"Observable match: {ind_match}")
65 
66         if self._is_whitelisted(observable.filter_regex, ind_match):
67             return {}
68 
69         return self._format_match(
70             OBSERVABLE_CLASS, observable.stix_target, ind_match, match_range
71         )
72 
73     def _parse(self, data: str) -> Dict[str, Dict]:
74         list_matches = {}
75 
76         # Defang text
77         data = ioc_finder.prepare_text(data)
78 
79         for observable in self.observable_list:
80             list_matches.update(self._extract_observable(observable, data))
81 
82         for entity in self.entity_list:
83             list_matches = self._extract_entity(entity, list_matches, data)
84 
85         self.helper.log_debug(f"Text: '{data}' -> extracts {list_matches}")
86         return list_matches
87 
88     def _parse_pdf(self, file_data: IO) -> Dict[str, Dict]:
89         parse_info = {}
90         try:
91             for page_layout in extract_pages(file_data):
92                 for element in page_layout:
93                     if isinstance(element, LTTextContainer):
94                         text = element.get_text()
95                         # Parsing with newlines has been deprecated
96                         no_newline_text = text.replace("\n", "")
97                         parse_info.update(self._parse(no_newline_text))
98 
99                 # TODO also extract information from images/figures using OCR
100                 # https://pdfminersix.readthedocs.io/en/latest/topic/converting_pdf_to_text.html#topic-pdf-to-text-layout
101 
102         except Exception as e:
103             logging.exception(f"Pdf Parsing Error: {e}")
104 
105         return parse_info
106 
107     def _parse_text(self, file_data: IO) -> Dict[str, Dict]:
108         parse_info = {}
109         for text in file_data.readlines():
110             parse_info.update(self._parse(text.decode("utf-8")))
111         return parse_info
112 
113     def run_parser(self, file_path: str, file_type: str) -> List[Dict]:
114         parsing_results = []
115 
116         file_parser = self.supported_file_types.get(file_type, None)
117         if not file_parser:
118             raise NotImplementedError(f"No parser available for file type {file_type}")
119 
120         if not os.path.isfile(file_path):
121             raise IOError(f"File path is not a file: {file_path}")
122 
123         self.helper.log_info(f"Parsing report {file_path} {file_type}")
124 
125         try:
126             with open(file_path, "rb") as file_data:
127                 parsing_results = file_parser(file_data)
128         except Exception as e:
129             logging.exception(f"Parsing Error: {e}")
130 
131         parsing_results = list(parsing_results.values())
132 
133         return parsing_results
134 
135     @staticmethod
136     def _format_match(
137         format_type: str, category: str, match: str, match_range: Tuple = (0, 0)
138     ) -> Dict:
139         return {
140             RESULT_FORMAT_TYPE: format_type,
141             RESULT_FORMAT_CATEGORY: category,
142             RESULT_FORMAT_MATCH: match,
143             RESULT_FORMAT_RANGE: match_range,
144         }
145 
146     @staticmethod
147     def _sco_present(
148         match_list: Dict, entity_range: Tuple, filter_sco_types: List
149     ) -> str:
150         for match_name, match_info in match_list.items():
151             if match_info[RESULT_FORMAT_CATEGORY] in filter_sco_types:
152                 if (
153                     match_info[RESULT_FORMAT_RANGE][0] <= entity_range[0]
154                     and entity_range[1] <= match_info[RESULT_FORMAT_RANGE][1]
155                 ):
156                     return match_name
157 
158         return ""
159 
160     def _extract_observable(self, observable: Observable, data: str) -> Dict:
161         list_matches = {}
162         if observable.detection_option == OBSERVABLE_DETECTION_CUSTOM_REGEX:
163             for regex in observable.regex:
164                 for match in regex.finditer(data):
165                     match_value = match.group()
166 
167                     ind_match = self._post_parse_observables(
168                         match_value, observable, match.span()
169                     )
170                     if ind_match:
171                         list_matches[match.group()] = ind_match
172 
173         elif observable.detection_option == OBSERVABLE_DETECTION_LIBRARY:
174             lookup_function = self.library_lookup.get(observable.stix_target, None)
175             if not lookup_function:
176                 self.helper.log_error(
177                     f"Selected library function is not implemented: {observable.iocfinder_function}"
178                 )
179                 return {}
180 
181             matches = lookup_function(data)
182 
183             for match in matches:
184                 start = data.index(str(match))
185                 ind_match = self._post_parse_observables(
186                     match, observable, (start, len(str(match)) + start)
187                 )
188                 if ind_match:
189                     list_matches[match] = ind_match
190 
191         return list_matches
192 
193     def _extract_entity(self, entity: Entity, list_matches: Dict, data: str) -> Dict:
194         regex_list = entity.regex
195 
196         observable_keys = []
197         end_index = set()
198         match_dict = {}
199         match_key = ""
200 
201         # Run all regexes for entity X
202         for regex in regex_list:
203             for match in regex.finditer(data):
204                 match_key = match.group()
205                 if match_key in match_dict:
206                     match_dict[match_key].append(match.span())
207                 else:
208                     match_dict[match_key] = [match.span()]
209 
210         # No maches for this entity
211         if len(match_dict) == 0:
212             return list_matches
213 
214         # Run through all matches for entity X and check if they are part of a domain
215         # yes -> skip
216         # no -> add index to end_index
217         for match, match_indices in match_dict.items():
218             for match_index in match_indices:
219                 skip_val = self._sco_present(
220                     list_matches, match_index, entity.omit_match_in
221                 )
222                 if skip_val:
223                     self.helper.log_debug(
224                         f"Skipping Entity '{match}', it is part of an omitted field '{entity.omit_match_in}' \"{skip_val}\""
225                     )
226                 else:
227                     self.helper.log_debug(
228                         f"Entity match: '{match}' of regex: '{regex_list}'"
229                     )
230                     end_index.add(match_index)
231                     if match in list_matches.keys():
232                         observable_keys.append(match)
233 
234         # Remove all observables which found the same information/Entity match
235         for observable_key in observable_keys:
236             del list_matches[observable_key]
237             self.helper.log_debug(
238                 f"Value {observable_key} is also matched by entity {entity.name}"
239             )
240 
241         # Check if entity was matched at least once in the text
242         # If yes, then add identity to match list
243         if end_index:
244             list_matches[match_key] = self._format_match(
245                 ENTITY_CLASS, entity.name, entity.stix_id
246             )
247 
248         return list_matches
249
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/internal-import-file/import-report/src/reportimporter/report_parser.py b/internal-import-file/import-report/src/reportimporter/report_parser.py
--- a/internal-import-file/import-report/src/reportimporter/report_parser.py
+++ b/internal-import-file/import-report/src/reportimporter/report_parser.py
@@ -181,9 +181,24 @@
             matches = lookup_function(data)
 
             for match in matches:
-                start = data.index(str(match))
+                match_str = str(match)
+                if match_str in data:
+                    start = data.index(match_str)
+                elif match_str in data.lower():
+                    self.helper.log_debug(
+                        f"External library manipulated the extracted value '{match_str}' from the "
+                        f"original text '{data}' to lower case"
+                    )
+                    start = data.lower().index(match_str)
+                else:
+                    self.helper.log_error(
+                        f"The extracted text '{match_str}' is not part of the original text '{data}'. "
+                        f"Please open a GitHub issue to report this problem!"
+                    )
+                    continue
+
                 ind_match = self._post_parse_observables(
-                    match, observable, (start, len(str(match)) + start)
+                    match, observable, (start, len(match_str) + start)
                 )
                 if ind_match:
                     list_matches[match] = ind_match

{"golden_diff": "diff --git a/internal-import-file/import-report/src/reportimporter/report_parser.py b/internal-import-file/import-report/src/reportimporter/report_parser.py\n--- a/internal-import-file/import-report/src/reportimporter/report_parser.py\n+++ b/internal-import-file/import-report/src/reportimporter/report_parser.py\n@@ -181,9 +181,24 @@\n             matches = lookup_function(data)\n \n             for match in matches:\n-                start = data.index(str(match))\n+                match_str = str(match)\n+                if match_str in data:\n+                    start = data.index(match_str)\n+                elif match_str in data.lower():\n+                    self.helper.log_debug(\n+                        f\"External library manipulated the extracted value '{match_str}' from the \"\n+                        f\"original text '{data}' to lower case\"\n+                    )\n+                    start = data.lower().index(match_str)\n+                else:\n+                    self.helper.log_error(\n+                        f\"The extracted text '{match_str}' is not part of the original text '{data}'. \"\n+                        f\"Please open a GitHub issue to report this problem!\"\n+                    )\n+                    continue\n+\n                 ind_match = self._post_parse_observables(\n-                    match, observable, (start, len(str(match)) + start)\n+                    match, observable, (start, len(match_str) + start)\n                 )\n                 if ind_match:\n                     list_matches[match] = ind_match\n", "issue": "[import-report] Trouble parsing domains / URLs with capital letters\n## Description\r\n\r\nimport-report appears to fail when parsing domains with capital letters. I've attached the txt file of domains I was trying to import below.  The import fails with the error \"'list' object has no attribute 'values'\". I played with the input file by removing domains that appeared to be causing issues, and the problematic domains were:\r\n\r\n- kAty197.chickenkiller[.]com\r\n- Soure7788.chickenkiller[.]com\r\n- Engaction[.]com\r\n\r\n## Environment\r\n\r\n1. OS (where OpenCTI server runs): Ubuntu 18\r\n2. OpenCTI version: OpenCTI 4.5.5\r\n3. OpenCTI client: frontend\r\n4. Other environment details: import-connector: rolling\r\n\r\n## Reproducible Steps\r\n\r\nSteps to create the smallest reproducible scenario:\r\n1. Import txt file with domains/URLs with capital letters\r\n\r\n## Expected Output\r\n\r\nDomains / URLs are parsed as observables\r\n\r\n## Actual Output\r\n\r\nImport-report connector crashes, no observables are parsed unless the capital letters become lowercase or the offending domains are removed\r\n \r\n## Additional information\r\n\r\nI've attached 3 files:\r\n\r\n- **AA21-200A_domains_failed.txt** : The original txt file, with domains including capital letters\r\n- **AA21-200A_domains_success.txt** : The revised txt file, with no capital letters\r\n- **errorlog.txt** : The error log, from one of the times when I tried to import a version of the original file (this was an iteration of troubleshooting, so the error log shows what happens when I input the txt file having removed the first two offending domains)\r\n\r\n[AA21-200A_domains_success.txt](https://github.com/OpenCTI-Platform/connectors/files/7101092/AA21-200A_domains_success.txt)\r\n[errorlog.txt](https://github.com/OpenCTI-Platform/connectors/files/7101093/errorlog.txt)\r\n[AA21-200A_domains_failed.txt](https://github.com/OpenCTI-Platform/connectors/files/7101094/AA21-200A_domains_failed.txt)\r\n\r\n\n", "before_files": [{"content": "import logging\nimport os\nfrom typing import Dict, List, Pattern, IO, Tuple\n\nimport ioc_finder\nfrom pdfminer.high_level import extract_pages\nfrom pdfminer.layout import LTTextContainer\nfrom pycti import OpenCTIConnectorHelper\nfrom reportimporter.constants import (\n    OBSERVABLE_CLASS,\n    ENTITY_CLASS,\n    RESULT_FORMAT_MATCH,\n    RESULT_FORMAT_TYPE,\n    RESULT_FORMAT_CATEGORY,\n    RESULT_FORMAT_RANGE,\n    MIME_PDF,\n    MIME_TXT,\n    OBSERVABLE_DETECTION_CUSTOM_REGEX,\n    OBSERVABLE_DETECTION_LIBRARY,\n)\nfrom reportimporter.models import Observable, Entity\nfrom reportimporter.util import library_mapping\n\n\nclass ReportParser(object):\n    \"\"\"\n    Report parser based on IOCParser\n    \"\"\"\n\n    def __init__(\n        self,\n        helper: OpenCTIConnectorHelper,\n        entity_list: List[Entity],\n        observable_list: List[Observable],\n    ):\n\n        self.helper = helper\n        self.entity_list = entity_list\n        self.observable_list = observable_list\n\n        # Disable INFO logging by pdfminer\n        logging.getLogger(\"pdfminer\").setLevel(logging.WARNING)\n\n        # Supported file types\n        self.supported_file_types = {\n            MIME_PDF: self._parse_pdf,\n            MIME_TXT: self._parse_text,\n        }\n\n        self.library_lookup = library_mapping()\n\n    def _is_whitelisted(self, regex_list: List[Pattern], ind_match: str):\n        for regex in regex_list:\n            self.helper.log_debug(f\"Filter regex '{regex}' for value '{ind_match}'\")\n            result = regex.search(ind_match)\n            if result:\n                self.helper.log_debug(f\"Value {ind_match} is whitelisted with {regex}\")\n                return True\n        return False\n\n    def _post_parse_observables(\n        self, ind_match: str, observable: Observable, match_range: Tuple\n    ) -> Dict:\n        self.helper.log_debug(f\"Observable match: {ind_match}\")\n\n        if self._is_whitelisted(observable.filter_regex, ind_match):\n            return {}\n\n        return self._format_match(\n            OBSERVABLE_CLASS, observable.stix_target, ind_match, match_range\n        )\n\n    def _parse(self, data: str) -> Dict[str, Dict]:\n        list_matches = {}\n\n        # Defang text\n        data = ioc_finder.prepare_text(data)\n\n        for observable in self.observable_list:\n            list_matches.update(self._extract_observable(observable, data))\n\n        for entity in self.entity_list:\n            list_matches = self._extract_entity(entity, list_matches, data)\n\n        self.helper.log_debug(f\"Text: '{data}' -> extracts {list_matches}\")\n        return list_matches\n\n    def _parse_pdf(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        try:\n            for page_layout in extract_pages(file_data):\n                for element in page_layout:\n                    if isinstance(element, LTTextContainer):\n                        text = element.get_text()\n                        # Parsing with newlines has been deprecated\n                        no_newline_text = text.replace(\"\\n\", \"\")\n                        parse_info.update(self._parse(no_newline_text))\n\n                # TODO also extract information from images/figures using OCR\n                # https://pdfminersix.readthedocs.io/en/latest/topic/converting_pdf_to_text.html#topic-pdf-to-text-layout\n\n        except Exception as e:\n            logging.exception(f\"Pdf Parsing Error: {e}\")\n\n        return parse_info\n\n    def _parse_text(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        for text in file_data.readlines():\n            parse_info.update(self._parse(text.decode(\"utf-8\")))\n        return parse_info\n\n    def run_parser(self, file_path: str, file_type: str) -> List[Dict]:\n        parsing_results = []\n\n        file_parser = self.supported_file_types.get(file_type, None)\n        if not file_parser:\n            raise NotImplementedError(f\"No parser available for file type {file_type}\")\n\n        if not os.path.isfile(file_path):\n            raise IOError(f\"File path is not a file: {file_path}\")\n\n        self.helper.log_info(f\"Parsing report {file_path} {file_type}\")\n\n        try:\n            with open(file_path, \"rb\") as file_data:\n                parsing_results = file_parser(file_data)\n        except Exception as e:\n            logging.exception(f\"Parsing Error: {e}\")\n\n        parsing_results = list(parsing_results.values())\n\n        return parsing_results\n\n    @staticmethod\n    def _format_match(\n        format_type: str, category: str, match: str, match_range: Tuple = (0, 0)\n    ) -> Dict:\n        return {\n            RESULT_FORMAT_TYPE: format_type,\n            RESULT_FORMAT_CATEGORY: category,\n            RESULT_FORMAT_MATCH: match,\n            RESULT_FORMAT_RANGE: match_range,\n        }\n\n    @staticmethod\n    def _sco_present(\n        match_list: Dict, entity_range: Tuple, filter_sco_types: List\n    ) -> str:\n        for match_name, match_info in match_list.items():\n            if match_info[RESULT_FORMAT_CATEGORY] in filter_sco_types:\n                if (\n                    match_info[RESULT_FORMAT_RANGE][0] <= entity_range[0]\n                    and entity_range[1] <= match_info[RESULT_FORMAT_RANGE][1]\n                ):\n                    return match_name\n\n        return \"\"\n\n    def _extract_observable(self, observable: Observable, data: str) -> Dict:\n        list_matches = {}\n        if observable.detection_option == OBSERVABLE_DETECTION_CUSTOM_REGEX:\n            for regex in observable.regex:\n                for match in regex.finditer(data):\n                    match_value = match.group()\n\n                    ind_match = self._post_parse_observables(\n                        match_value, observable, match.span()\n                    )\n                    if ind_match:\n                        list_matches[match.group()] = ind_match\n\n        elif observable.detection_option == OBSERVABLE_DETECTION_LIBRARY:\n            lookup_function = self.library_lookup.get(observable.stix_target, None)\n            if not lookup_function:\n                self.helper.log_error(\n                    f\"Selected library function is not implemented: {observable.iocfinder_function}\"\n                )\n                return {}\n\n            matches = lookup_function(data)\n\n            for match in matches:\n                start = data.index(str(match))\n                ind_match = self._post_parse_observables(\n                    match, observable, (start, len(str(match)) + start)\n                )\n                if ind_match:\n                    list_matches[match] = ind_match\n\n        return list_matches\n\n    def _extract_entity(self, entity: Entity, list_matches: Dict, data: str) -> Dict:\n        regex_list = entity.regex\n\n        observable_keys = []\n        end_index = set()\n        match_dict = {}\n        match_key = \"\"\n\n        # Run all regexes for entity X\n        for regex in regex_list:\n            for match in regex.finditer(data):\n                match_key = match.group()\n                if match_key in match_dict:\n                    match_dict[match_key].append(match.span())\n                else:\n                    match_dict[match_key] = [match.span()]\n\n        # No maches for this entity\n        if len(match_dict) == 0:\n            return list_matches\n\n        # Run through all matches for entity X and check if they are part of a domain\n        # yes -> skip\n        # no -> add index to end_index\n        for match, match_indices in match_dict.items():\n            for match_index in match_indices:\n                skip_val = self._sco_present(\n                    list_matches, match_index, entity.omit_match_in\n                )\n                if skip_val:\n                    self.helper.log_debug(\n                        f\"Skipping Entity '{match}', it is part of an omitted field '{entity.omit_match_in}' \\\"{skip_val}\\\"\"\n                    )\n                else:\n                    self.helper.log_debug(\n                        f\"Entity match: '{match}' of regex: '{regex_list}'\"\n                    )\n                    end_index.add(match_index)\n                    if match in list_matches.keys():\n                        observable_keys.append(match)\n\n        # Remove all observables which found the same information/Entity match\n        for observable_key in observable_keys:\n            del list_matches[observable_key]\n            self.helper.log_debug(\n                f\"Value {observable_key} is also matched by entity {entity.name}\"\n            )\n\n        # Check if entity was matched at least once in the text\n        # If yes, then add identity to match list\n        if end_index:\n            list_matches[match_key] = self._format_match(\n                ENTITY_CLASS, entity.name, entity.stix_id\n            )\n\n        return list_matches\n", "path": "internal-import-file/import-report/src/reportimporter/report_parser.py"}], "after_files": [{"content": "import logging\nimport os\nfrom typing import Dict, List, Pattern, IO, Tuple\n\nimport ioc_finder\nfrom pdfminer.high_level import extract_pages\nfrom pdfminer.layout import LTTextContainer\nfrom pycti import OpenCTIConnectorHelper\nfrom reportimporter.constants import (\n    OBSERVABLE_CLASS,\n    ENTITY_CLASS,\n    RESULT_FORMAT_MATCH,\n    RESULT_FORMAT_TYPE,\n    RESULT_FORMAT_CATEGORY,\n    RESULT_FORMAT_RANGE,\n    MIME_PDF,\n    MIME_TXT,\n    OBSERVABLE_DETECTION_CUSTOM_REGEX,\n    OBSERVABLE_DETECTION_LIBRARY,\n)\nfrom reportimporter.models import Observable, Entity\nfrom reportimporter.util import library_mapping\n\n\nclass ReportParser(object):\n    \"\"\"\n    Report parser based on IOCParser\n    \"\"\"\n\n    def __init__(\n        self,\n        helper: OpenCTIConnectorHelper,\n        entity_list: List[Entity],\n        observable_list: List[Observable],\n    ):\n\n        self.helper = helper\n        self.entity_list = entity_list\n        self.observable_list = observable_list\n\n        # Disable INFO logging by pdfminer\n        logging.getLogger(\"pdfminer\").setLevel(logging.WARNING)\n\n        # Supported file types\n        self.supported_file_types = {\n            MIME_PDF: self._parse_pdf,\n            MIME_TXT: self._parse_text,\n        }\n\n        self.library_lookup = library_mapping()\n\n    def _is_whitelisted(self, regex_list: List[Pattern], ind_match: str):\n        for regex in regex_list:\n            self.helper.log_debug(f\"Filter regex '{regex}' for value '{ind_match}'\")\n            result = regex.search(ind_match)\n            if result:\n                self.helper.log_debug(f\"Value {ind_match} is whitelisted with {regex}\")\n                return True\n        return False\n\n    def _post_parse_observables(\n        self, ind_match: str, observable: Observable, match_range: Tuple\n    ) -> Dict:\n        self.helper.log_debug(f\"Observable match: {ind_match}\")\n\n        if self._is_whitelisted(observable.filter_regex, ind_match):\n            return {}\n\n        return self._format_match(\n            OBSERVABLE_CLASS, observable.stix_target, ind_match, match_range\n        )\n\n    def _parse(self, data: str) -> Dict[str, Dict]:\n        list_matches = {}\n\n        # Defang text\n        data = ioc_finder.prepare_text(data)\n\n        for observable in self.observable_list:\n            list_matches.update(self._extract_observable(observable, data))\n\n        for entity in self.entity_list:\n            list_matches = self._extract_entity(entity, list_matches, data)\n\n        self.helper.log_debug(f\"Text: '{data}' -> extracts {list_matches}\")\n        return list_matches\n\n    def _parse_pdf(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        try:\n            for page_layout in extract_pages(file_data):\n                for element in page_layout:\n                    if isinstance(element, LTTextContainer):\n                        text = element.get_text()\n                        # Parsing with newlines has been deprecated\n                        no_newline_text = text.replace(\"\\n\", \"\")\n                        parse_info.update(self._parse(no_newline_text))\n\n                # TODO also extract information from images/figures using OCR\n                # https://pdfminersix.readthedocs.io/en/latest/topic/converting_pdf_to_text.html#topic-pdf-to-text-layout\n\n        except Exception as e:\n            logging.exception(f\"Pdf Parsing Error: {e}\")\n\n        return parse_info\n\n    def _parse_text(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        for text in file_data.readlines():\n            parse_info.update(self._parse(text.decode(\"utf-8\")))\n        return parse_info\n\n    def run_parser(self, file_path: str, file_type: str) -> List[Dict]:\n        parsing_results = []\n\n        file_parser = self.supported_file_types.get(file_type, None)\n        if not file_parser:\n            raise NotImplementedError(f\"No parser available for file type {file_type}\")\n\n        if not os.path.isfile(file_path):\n            raise IOError(f\"File path is not a file: {file_path}\")\n\n        self.helper.log_info(f\"Parsing report {file_path} {file_type}\")\n\n        try:\n            with open(file_path, \"rb\") as file_data:\n                parsing_results = file_parser(file_data)\n        except Exception as e:\n            logging.exception(f\"Parsing Error: {e}\")\n\n        parsing_results = list(parsing_results.values())\n\n        return parsing_results\n\n    @staticmethod\n    def _format_match(\n        format_type: str, category: str, match: str, match_range: Tuple = (0, 0)\n    ) -> Dict:\n        return {\n            RESULT_FORMAT_TYPE: format_type,\n            RESULT_FORMAT_CATEGORY: category,\n            RESULT_FORMAT_MATCH: match,\n            RESULT_FORMAT_RANGE: match_range,\n        }\n\n    @staticmethod\n    def _sco_present(\n        match_list: Dict, entity_range: Tuple, filter_sco_types: List\n    ) -> str:\n        for match_name, match_info in match_list.items():\n            if match_info[RESULT_FORMAT_CATEGORY] in filter_sco_types:\n                if (\n                    match_info[RESULT_FORMAT_RANGE][0] <= entity_range[0]\n                    and entity_range[1] <= match_info[RESULT_FORMAT_RANGE][1]\n                ):\n                    return match_name\n\n        return \"\"\n\n    def _extract_observable(self, observable: Observable, data: str) -> Dict:\n        list_matches = {}\n        if observable.detection_option == OBSERVABLE_DETECTION_CUSTOM_REGEX:\n            for regex in observable.regex:\n                for match in regex.finditer(data):\n                    match_value = match.group()\n\n                    ind_match = self._post_parse_observables(\n                        match_value, observable, match.span()\n                    )\n                    if ind_match:\n                        list_matches[match.group()] = ind_match\n\n        elif observable.detection_option == OBSERVABLE_DETECTION_LIBRARY:\n            lookup_function = self.library_lookup.get(observable.stix_target, None)\n            if not lookup_function:\n                self.helper.log_error(\n                    f\"Selected library function is not implemented: {observable.iocfinder_function}\"\n                )\n                return {}\n\n            matches = lookup_function(data)\n\n            for match in matches:\n                match_str = str(match)\n                if match_str in data:\n                    start = data.index(match_str)\n                elif match_str in data.lower():\n                    self.helper.log_debug(\n                        f\"External library manipulated the extracted value '{match_str}' from the \"\n                        f\"original text '{data}' to lower case\"\n                    )\n                    start = data.lower().index(match_str)\n                else:\n                    self.helper.log_error(\n                        f\"The extracted text '{match_str}' is not part of the original text '{data}'. \"\n                        f\"Please open a GitHub issue to report this problem!\"\n                    )\n                    continue\n\n                ind_match = self._post_parse_observables(\n                    match, observable, (start, len(match_str) + start)\n                )\n                if ind_match:\n                    list_matches[match] = ind_match\n\n        return list_matches\n\n    def _extract_entity(self, entity: Entity, list_matches: Dict, data: str) -> Dict:\n        regex_list = entity.regex\n\n        observable_keys = []\n        end_index = set()\n        match_dict = {}\n        match_key = \"\"\n\n        # Run all regexes for entity X\n        for regex in regex_list:\n            for match in regex.finditer(data):\n                match_key = match.group()\n                if match_key in match_dict:\n                    match_dict[match_key].append(match.span())\n                else:\n                    match_dict[match_key] = [match.span()]\n\n        # No maches for this entity\n        if len(match_dict) == 0:\n            return list_matches\n\n        # Run through all matches for entity X and check if they are part of a domain\n        # yes -> skip\n        # no -> add index to end_index\n        for match, match_indices in match_dict.items():\n            for match_index in match_indices:\n                skip_val = self._sco_present(\n                    list_matches, match_index, entity.omit_match_in\n                )\n                if skip_val:\n                    self.helper.log_debug(\n                        f\"Skipping Entity '{match}', it is part of an omitted field '{entity.omit_match_in}' \\\"{skip_val}\\\"\"\n                    )\n                else:\n                    self.helper.log_debug(\n                        f\"Entity match: '{match}' of regex: '{regex_list}'\"\n                    )\n                    end_index.add(match_index)\n                    if match in list_matches.keys():\n                        observable_keys.append(match)\n\n        # Remove all observables which found the same information/Entity match\n        for observable_key in observable_keys:\n            del list_matches[observable_key]\n            self.helper.log_debug(\n                f\"Value {observable_key} is also matched by entity {entity.name}\"\n            )\n\n        # Check if entity was matched at least once in the text\n        # If yes, then add identity to match list\n        if end_index:\n            list_matches[match_key] = self._format_match(\n                ENTITY_CLASS, entity.name, entity.stix_id\n            )\n\n        return list_matches\n", "path": "internal-import-file/import-report/src/reportimporter/report_parser.py"}]}