We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[VXVault] Connector imports garbage data (HTML tags and blank events)
## Description

The VXVault connector is importing garbage data as indicators and observables, including events like `<pre>`, </pre>` and blank entities. 

## Environment

1. OS (where OpenCTI server runs): Docker on Ubuntu 20 
2. OpenCTI version: 5.5.2
3. OpenCTI client: connectors/python

## Expected Output
Connector should download the URLs from the VXVault threat feed here: http://vxvault.net/URL_List.php and import them into OpenCTI as Observables and Indicators.

## Actual Output
The connector does import the URLs, but it fails to parse the page correctly and imports some of the HTML tags as well by accident, leading to garbage data like Indicators with names and values as just `<pre>`

## Additional information
I discussed this and a couple of other related issues with the VXVault connector + data with Samuel Hassine in the slack channel: https://filigran-community.slack.com/archives/CHZC2D38C/p1673599524232109

There are three related issues - I will raise a GH issue for two of them and then submit a proposed PR to fix both:
1 - Connector imports garbage data (HTML tags)
2 - Connector leaves trailing white-space at the end of Indicator/Observable name and value (Raised as #974)
3 - VXVault entities do not show up in bulk search, which is caused by issue 2, as bulk-search only does an exact match currently and can't match on the trailing `\n` characters. The developers are aware of this, so I will not raise an issue for it.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `external-import/vxvault/src/vxvault.py`
Content:
```
1 import os
2 import ssl
3 import sys
4 import time
5 import urllib.request
6 from datetime import datetime
7 
8 import certifi
9 import yaml
10 from pycti import OpenCTIConnectorHelper, get_config_variable
11 from stix2 import TLP_WHITE, URL, Bundle, ExternalReference
12 
13 
14 class VXVault:
15     def __init__(self):
16         # Instantiate the connector helper from config
17         config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
18         config = (
19             yaml.load(open(config_file_path), Loader=yaml.FullLoader)
20             if os.path.isfile(config_file_path)
21             else {}
22         )
23         self.helper = OpenCTIConnectorHelper(config)
24         # Extra config
25         self.vxvault_url = get_config_variable(
26             "VXVAULT_URL", ["vxvault", "url"], config
27         )
28         self.vxvault_interval = get_config_variable(
29             "VXVAULT_INTERVAL", ["vxvault", "interval"], config, True
30         )
31         self.create_indicators = get_config_variable(
32             "VXVAULT_CREATE_INDICATORS",
33             ["vxvault", "create_indicators"],
34             config,
35             False,
36             True,
37         )
38         self.update_existing_data = get_config_variable(
39             "CONNECTOR_UPDATE_EXISTING_DATA",
40             ["connector", "update_existing_data"],
41             config,
42         )
43         self.identity = self.helper.api.identity.create(
44             type="Organization",
45             name="VX Vault",
46             description="VX Vault is providing URLs of potential malicious payload.",
47         )
48 
49     def get_interval(self):
50         return int(self.vxvault_interval) * 60 * 60 * 24
51 
52     def next_run(self, seconds):
53         return
54 
55     def run(self):
56         self.helper.log_info("Fetching VXVault dataset...")
57         while True:
58             try:
59                 # Get the current timestamp and check
60                 timestamp = int(time.time())
61                 current_state = self.helper.get_state()
62                 if current_state is not None and "last_run" in current_state:
63                     last_run = current_state["last_run"]
64                     self.helper.log_info(
65                         "Connector last run: "
66                         + datetime.utcfromtimestamp(last_run).strftime(
67                             "%Y-%m-%d %H:%M:%S"
68                         )
69                     )
70                 else:
71                     last_run = None
72                     self.helper.log_info("Connector has never run")
73                 # If the last_run is more than interval-1 day
74                 if last_run is None or (
75                     (timestamp - last_run)
76                     > ((int(self.vxvault_interval) - 1) * 60 * 60 * 24)
77                 ):
78                     self.helper.log_info("Connector will run!")
79                     now = datetime.utcfromtimestamp(timestamp)
80                     friendly_name = "VXVault run @ " + now.strftime("%Y-%m-%d %H:%M:%S")
81                     work_id = self.helper.api.work.initiate_work(
82                         self.helper.connect_id, friendly_name
83                     )
84                     try:
85                         response = urllib.request.urlopen(
86                             self.vxvault_url,
87                             context=ssl.create_default_context(cafile=certifi.where()),
88                         )
89                         image = response.read()
90                         with open(
91                             os.path.dirname(os.path.abspath(__file__)) + "/data.txt",
92                             "wb",
93                         ) as file:
94                             file.write(image)
95                         count = 0
96                         bundle_objects = []
97                         with open(
98                             os.path.dirname(os.path.abspath(__file__)) + "/data.txt"
99                         ) as fp:
100                             for line in fp:
101                                 count += 1
102                                 if count <= 3:
103                                     continue
104                                 external_reference = ExternalReference(
105                                     source_name="VX Vault",
106                                     url="http://vxvault.net",
107                                     description="VX Vault repository URL",
108                                 )
109                                 stix_observable = URL(
110                                     value=line,
111                                     object_marking_refs=[TLP_WHITE],
112                                     custom_properties={
113                                         "description": "VX Vault URL",
114                                         "x_opencti_score": 80,
115                                         "created_by_ref": self.identity["standard_id"],
116                                         "x_opencti_create_indicator": self.create_indicators,
117                                         "external_references": [external_reference],
118                                     },
119                                 )
120                                 bundle_objects.append(stix_observable)
121                         bundle = Bundle(
122                             objects=bundle_objects, allow_custom=True
123                         ).serialize()
124                         self.helper.send_stix2_bundle(
125                             bundle,
126                             update=self.update_existing_data,
127                             work_id=work_id,
128                         )
129                         if os.path.exists(
130                             os.path.dirname(os.path.abspath(__file__)) + "/data.txt"
131                         ):
132                             os.remove(
133                                 os.path.dirname(os.path.abspath(__file__)) + "/data.txt"
134                             )
135                     except Exception as e:
136                         self.helper.log_error(str(e))
137                     # Store the current timestamp as a last run
138                     message = "Connector successfully run, storing last_run as " + str(
139                         timestamp
140                     )
141                     self.helper.log_info(message)
142                     self.helper.set_state({"last_run": timestamp})
143                     self.helper.api.work.to_processed(work_id, message)
144                     self.helper.log_info(
145                         "Last_run stored, next run in: "
146                         + str(round(self.get_interval() / 60 / 60 / 24, 2))
147                         + " days"
148                     )
149                 else:
150                     new_interval = self.get_interval() - (timestamp - last_run)
151                     self.helper.log_info(
152                         "Connector will not run, next run in: "
153                         + str(round(new_interval / 60 / 60 / 24, 2))
154                         + " days"
155                     )
156             except (KeyboardInterrupt, SystemExit):
157                 self.helper.log_info("Connector stop")
158                 sys.exit(0)
159             except Exception as e:
160                 self.helper.log_error(str(e))
161 
162             if self.helper.connect_run_and_terminate:
163                 self.helper.log_info("Connector stop")
164                 sys.exit(0)
165 
166             time.sleep(60)
167 
168 
169 if __name__ == "__main__":
170     try:
171         VXVaultConnector = VXVault()
172         VXVaultConnector.run()
173     except Exception as e:
174         print(e)
175         time.sleep(10)
176         sys.exit(0)
177
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/external-import/vxvault/src/vxvault.py b/external-import/vxvault/src/vxvault.py
--- a/external-import/vxvault/src/vxvault.py
+++ b/external-import/vxvault/src/vxvault.py
@@ -1,4 +1,5 @@
 import os
+import re
 import ssl
 import sys
 import time
@@ -7,6 +8,7 @@
 
 import certifi
 import yaml
+
 from pycti import OpenCTIConnectorHelper, get_config_variable
 from stix2 import TLP_WHITE, URL, Bundle, ExternalReference
 
@@ -101,6 +103,13 @@
                                 count += 1
                                 if count <= 3:
                                     continue
+                                line=line.strip()
+                                matchHtmlTag = re.search(r'^<\/?\w+>', line)
+                                if matchHtmlTag:
+                                    continue
+                                matchBlankLine = re.search(r'^\s*$', line)
+                                if matchBlankLine:
+                                    continue
                                 external_reference = ExternalReference(
                                     source_name="VX Vault",
                                     url="http://vxvault.net",

{"golden_diff": "diff --git a/external-import/vxvault/src/vxvault.py b/external-import/vxvault/src/vxvault.py\n--- a/external-import/vxvault/src/vxvault.py\n+++ b/external-import/vxvault/src/vxvault.py\n@@ -1,4 +1,5 @@\n import os\n+import re\n import ssl\n import sys\n import time\n@@ -7,6 +8,7 @@\n \n import certifi\n import yaml\n+\n from pycti import OpenCTIConnectorHelper, get_config_variable\n from stix2 import TLP_WHITE, URL, Bundle, ExternalReference\n \n@@ -101,6 +103,13 @@\n                                 count += 1\n                                 if count <= 3:\n                                     continue\n+                                line=line.strip()\n+                                matchHtmlTag = re.search(r'^<\\/?\\w+>', line)\n+                                if matchHtmlTag:\n+                                    continue\n+                                matchBlankLine = re.search(r'^\\s*$', line)\n+                                if matchBlankLine:\n+                                    continue\n                                 external_reference = ExternalReference(\n                                     source_name=\"VX Vault\",\n                                     url=\"http://vxvault.net\",\n", "issue": "[VXVault] Connector imports garbage data (HTML tags and blank events)\n## Description\r\n\r\nThe VXVault connector is importing garbage data as indicators and observables, including events like `<pre>`, </pre>` and blank entities. \r\n\r\n## Environment\r\n\r\n1. OS (where OpenCTI server runs): Docker on Ubuntu 20 \r\n2. OpenCTI version: 5.5.2\r\n3. OpenCTI client: connectors/python\r\n\r\n## Expected Output\r\nConnector should download the URLs from the VXVault threat feed here: http://vxvault.net/URL_List.php and import them into OpenCTI as Observables and Indicators.\r\n\r\n## Actual Output\r\nThe connector does import the URLs, but it fails to parse the page correctly and imports some of the HTML tags as well by accident, leading to garbage data like Indicators with names and values as just `<pre>`\r\n\r\n## Additional information\r\nI discussed this and a couple of other related issues with the VXVault connector + data with Samuel Hassine in the slack channel: https://filigran-community.slack.com/archives/CHZC2D38C/p1673599524232109\r\n\r\nThere are three related issues - I will raise a GH issue for two of them and then submit a proposed PR to fix both:\r\n1 - Connector imports garbage data (HTML tags)\r\n2 - Connector leaves trailing white-space at the end of Indicator/Observable name and value (Raised as #974)\r\n3 - VXVault entities do not show up in bulk search, which is caused by issue 2, as bulk-search only does an exact match currently and can't match on the trailing `\\n` characters. The developers are aware of this, so I will not raise an issue for it.\n", "before_files": [{"content": "import os\nimport ssl\nimport sys\nimport time\nimport urllib.request\nfrom datetime import datetime\n\nimport certifi\nimport yaml\nfrom pycti import OpenCTIConnectorHelper, get_config_variable\nfrom stix2 import TLP_WHITE, URL, Bundle, ExternalReference\n\n\nclass VXVault:\n    def __init__(self):\n        # Instantiate the connector helper from config\n        config_file_path = os.path.dirname(os.path.abspath(__file__)) + \"/config.yml\"\n        config = (\n            yaml.load(open(config_file_path), Loader=yaml.FullLoader)\n            if os.path.isfile(config_file_path)\n            else {}\n        )\n        self.helper = OpenCTIConnectorHelper(config)\n        # Extra config\n        self.vxvault_url = get_config_variable(\n            \"VXVAULT_URL\", [\"vxvault\", \"url\"], config\n        )\n        self.vxvault_interval = get_config_variable(\n            \"VXVAULT_INTERVAL\", [\"vxvault\", \"interval\"], config, True\n        )\n        self.create_indicators = get_config_variable(\n            \"VXVAULT_CREATE_INDICATORS\",\n            [\"vxvault\", \"create_indicators\"],\n            config,\n            False,\n            True,\n        )\n        self.update_existing_data = get_config_variable(\n            \"CONNECTOR_UPDATE_EXISTING_DATA\",\n            [\"connector\", \"update_existing_data\"],\n            config,\n        )\n        self.identity = self.helper.api.identity.create(\n            type=\"Organization\",\n            name=\"VX Vault\",\n            description=\"VX Vault is providing URLs of potential malicious payload.\",\n        )\n\n    def get_interval(self):\n        return int(self.vxvault_interval) * 60 * 60 * 24\n\n    def next_run(self, seconds):\n        return\n\n    def run(self):\n        self.helper.log_info(\"Fetching VXVault dataset...\")\n        while True:\n            try:\n                # Get the current timestamp and check\n                timestamp = int(time.time())\n                current_state = self.helper.get_state()\n                if current_state is not None and \"last_run\" in current_state:\n                    last_run = current_state[\"last_run\"]\n                    self.helper.log_info(\n                        \"Connector last run: \"\n                        + datetime.utcfromtimestamp(last_run).strftime(\n                            \"%Y-%m-%d %H:%M:%S\"\n                        )\n                    )\n                else:\n                    last_run = None\n                    self.helper.log_info(\"Connector has never run\")\n                # If the last_run is more than interval-1 day\n                if last_run is None or (\n                    (timestamp - last_run)\n                    > ((int(self.vxvault_interval) - 1) * 60 * 60 * 24)\n                ):\n                    self.helper.log_info(\"Connector will run!\")\n                    now = datetime.utcfromtimestamp(timestamp)\n                    friendly_name = \"VXVault run @ \" + now.strftime(\"%Y-%m-%d %H:%M:%S\")\n                    work_id = self.helper.api.work.initiate_work(\n                        self.helper.connect_id, friendly_name\n                    )\n                    try:\n                        response = urllib.request.urlopen(\n                            self.vxvault_url,\n                            context=ssl.create_default_context(cafile=certifi.where()),\n                        )\n                        image = response.read()\n                        with open(\n                            os.path.dirname(os.path.abspath(__file__)) + \"/data.txt\",\n                            \"wb\",\n                        ) as file:\n                            file.write(image)\n                        count = 0\n                        bundle_objects = []\n                        with open(\n                            os.path.dirname(os.path.abspath(__file__)) + \"/data.txt\"\n                        ) as fp:\n                            for line in fp:\n                                count += 1\n                                if count <= 3:\n                                    continue\n                                external_reference = ExternalReference(\n                                    source_name=\"VX Vault\",\n                                    url=\"http://vxvault.net\",\n                                    description=\"VX Vault repository URL\",\n                                )\n                                stix_observable = URL(\n                                    value=line,\n                                    object_marking_refs=[TLP_WHITE],\n                                    custom_properties={\n                                        \"description\": \"VX Vault URL\",\n                                        \"x_opencti_score\": 80,\n                                        \"created_by_ref\": self.identity[\"standard_id\"],\n                                        \"x_opencti_create_indicator\": self.create_indicators,\n                                        \"external_references\": [external_reference],\n                                    },\n                                )\n                                bundle_objects.append(stix_observable)\n                        bundle = Bundle(\n                            objects=bundle_objects, allow_custom=True\n                        ).serialize()\n                        self.helper.send_stix2_bundle(\n                            bundle,\n                            update=self.update_existing_data,\n                            work_id=work_id,\n                        )\n                        if os.path.exists(\n                            os.path.dirname(os.path.abspath(__file__)) + \"/data.txt\"\n                        ):\n                            os.remove(\n                                os.path.dirname(os.path.abspath(__file__)) + \"/data.txt\"\n                            )\n                    except Exception as e:\n                        self.helper.log_error(str(e))\n                    # Store the current timestamp as a last run\n                    message = \"Connector successfully run, storing last_run as \" + str(\n                        timestamp\n                    )\n                    self.helper.log_info(message)\n                    self.helper.set_state({\"last_run\": timestamp})\n                    self.helper.api.work.to_processed(work_id, message)\n                    self.helper.log_info(\n                        \"Last_run stored, next run in: \"\n                        + str(round(self.get_interval() / 60 / 60 / 24, 2))\n                        + \" days\"\n                    )\n                else:\n                    new_interval = self.get_interval() - (timestamp - last_run)\n                    self.helper.log_info(\n                        \"Connector will not run, next run in: \"\n                        + str(round(new_interval / 60 / 60 / 24, 2))\n                        + \" days\"\n                    )\n            except (KeyboardInterrupt, SystemExit):\n                self.helper.log_info(\"Connector stop\")\n                sys.exit(0)\n            except Exception as e:\n                self.helper.log_error(str(e))\n\n            if self.helper.connect_run_and_terminate:\n                self.helper.log_info(\"Connector stop\")\n                sys.exit(0)\n\n            time.sleep(60)\n\n\nif __name__ == \"__main__\":\n    try:\n        VXVaultConnector = VXVault()\n        VXVaultConnector.run()\n    except Exception as e:\n        print(e)\n        time.sleep(10)\n        sys.exit(0)\n", "path": "external-import/vxvault/src/vxvault.py"}], "after_files": [{"content": "import os\nimport re\nimport ssl\nimport sys\nimport time\nimport urllib.request\nfrom datetime import datetime\n\nimport certifi\nimport yaml\n\nfrom pycti import OpenCTIConnectorHelper, get_config_variable\nfrom stix2 import TLP_WHITE, URL, Bundle, ExternalReference\n\n\nclass VXVault:\n    def __init__(self):\n        # Instantiate the connector helper from config\n        config_file_path = os.path.dirname(os.path.abspath(__file__)) + \"/config.yml\"\n        config = (\n            yaml.load(open(config_file_path), Loader=yaml.FullLoader)\n            if os.path.isfile(config_file_path)\n            else {}\n        )\n        self.helper = OpenCTIConnectorHelper(config)\n        # Extra config\n        self.vxvault_url = get_config_variable(\n            \"VXVAULT_URL\", [\"vxvault\", \"url\"], config\n        )\n        self.vxvault_interval = get_config_variable(\n            \"VXVAULT_INTERVAL\", [\"vxvault\", \"interval\"], config, True\n        )\n        self.create_indicators = get_config_variable(\n            \"VXVAULT_CREATE_INDICATORS\",\n            [\"vxvault\", \"create_indicators\"],\n            config,\n            False,\n            True,\n        )\n        self.update_existing_data = get_config_variable(\n            \"CONNECTOR_UPDATE_EXISTING_DATA\",\n            [\"connector\", \"update_existing_data\"],\n            config,\n        )\n        self.identity = self.helper.api.identity.create(\n            type=\"Organization\",\n            name=\"VX Vault\",\n            description=\"VX Vault is providing URLs of potential malicious payload.\",\n        )\n\n    def get_interval(self):\n        return int(self.vxvault_interval) * 60 * 60 * 24\n\n    def next_run(self, seconds):\n        return\n\n    def run(self):\n        self.helper.log_info(\"Fetching VXVault dataset...\")\n        while True:\n            try:\n                # Get the current timestamp and check\n                timestamp = int(time.time())\n                current_state = self.helper.get_state()\n                if current_state is not None and \"last_run\" in current_state:\n                    last_run = current_state[\"last_run\"]\n                    self.helper.log_info(\n                        \"Connector last run: \"\n                        + datetime.utcfromtimestamp(last_run).strftime(\n                            \"%Y-%m-%d %H:%M:%S\"\n                        )\n                    )\n                else:\n                    last_run = None\n                    self.helper.log_info(\"Connector has never run\")\n                # If the last_run is more than interval-1 day\n                if last_run is None or (\n                    (timestamp - last_run)\n                    > ((int(self.vxvault_interval) - 1) * 60 * 60 * 24)\n                ):\n                    self.helper.log_info(\"Connector will run!\")\n                    now = datetime.utcfromtimestamp(timestamp)\n                    friendly_name = \"VXVault run @ \" + now.strftime(\"%Y-%m-%d %H:%M:%S\")\n                    work_id = self.helper.api.work.initiate_work(\n                        self.helper.connect_id, friendly_name\n                    )\n                    try:\n                        response = urllib.request.urlopen(\n                            self.vxvault_url,\n                            context=ssl.create_default_context(cafile=certifi.where()),\n                        )\n                        image = response.read()\n                        with open(\n                            os.path.dirname(os.path.abspath(__file__)) + \"/data.txt\",\n                            \"wb\",\n                        ) as file:\n                            file.write(image)\n                        count = 0\n                        bundle_objects = []\n                        with open(\n                            os.path.dirname(os.path.abspath(__file__)) + \"/data.txt\"\n                        ) as fp:\n                            for line in fp:\n                                count += 1\n                                if count <= 3:\n                                    continue\n                                line=line.strip()\n                                matchHtmlTag = re.search(r'^<\\/?\\w+>', line)\n                                if matchHtmlTag:\n                                    continue\n                                matchBlankLine = re.search(r'^\\s*$', line)\n                                if matchBlankLine:\n                                    continue\n                                external_reference = ExternalReference(\n                                    source_name=\"VX Vault\",\n                                    url=\"http://vxvault.net\",\n                                    description=\"VX Vault repository URL\",\n                                )\n                                stix_observable = URL(\n                                    value=line,\n                                    object_marking_refs=[TLP_WHITE],\n                                    custom_properties={\n                                        \"description\": \"VX Vault URL\",\n                                        \"x_opencti_score\": 80,\n                                        \"created_by_ref\": self.identity[\"standard_id\"],\n                                        \"x_opencti_create_indicator\": self.create_indicators,\n                                        \"external_references\": [external_reference],\n                                    },\n                                )\n                                bundle_objects.append(stix_observable)\n                        bundle = Bundle(\n                            objects=bundle_objects, allow_custom=True\n                        ).serialize()\n                        self.helper.send_stix2_bundle(\n                            bundle,\n                            update=self.update_existing_data,\n                            work_id=work_id,\n                        )\n                        if os.path.exists(\n                            os.path.dirname(os.path.abspath(__file__)) + \"/data.txt\"\n                        ):\n                            os.remove(\n                                os.path.dirname(os.path.abspath(__file__)) + \"/data.txt\"\n                            )\n                    except Exception as e:\n                        self.helper.log_error(str(e))\n                    # Store the current timestamp as a last run\n                    message = \"Connector successfully run, storing last_run as \" + str(\n                        timestamp\n                    )\n                    self.helper.log_info(message)\n                    self.helper.set_state({\"last_run\": timestamp})\n                    self.helper.api.work.to_processed(work_id, message)\n                    self.helper.log_info(\n                        \"Last_run stored, next run in: \"\n                        + str(round(self.get_interval() / 60 / 60 / 24, 2))\n                        + \" days\"\n                    )\n                else:\n                    new_interval = self.get_interval() - (timestamp - last_run)\n                    self.helper.log_info(\n                        \"Connector will not run, next run in: \"\n                        + str(round(new_interval / 60 / 60 / 24, 2))\n                        + \" days\"\n                    )\n            except (KeyboardInterrupt, SystemExit):\n                self.helper.log_info(\"Connector stop\")\n                sys.exit(0)\n            except Exception as e:\n                self.helper.log_error(str(e))\n\n            if self.helper.connect_run_and_terminate:\n                self.helper.log_info(\"Connector stop\")\n                sys.exit(0)\n\n            time.sleep(60)\n\n\nif __name__ == \"__main__\":\n    try:\n        VXVaultConnector = VXVault()\n        VXVaultConnector.run()\n    except Exception as e:\n        print(e)\n        time.sleep(10)\n        sys.exit(0)\n", "path": "external-import/vxvault/src/vxvault.py"}]}