We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[urllib imports]  Multiple urllib imports are missing
**Small description**
Missing imports for `urllib.error` and `urllib.parse`


**Expected result**
No stack traces from missing imports.


**Actual result with screenshot**

First Trace
```
Traceback (most recent call last):
  File "/usr/lib/python3.9/site-packages/visidata/threads.py", line 200, in _toplevelTryFunc
    t.status = func(*args, **kwargs)
  File "/usr/lib/python3.9/site-packages/visidata/threads.py", line 219, in _func
    func(*args, **kwargs)
  File "/usr/lib/python3.9/site-packages/visidata/plugins.py", line 97, in reload
    except urllib.error.URLError as e:
AttributeError: module 'urllib' has no attribute 'error'
```

Second trace
```
Traceback (most recent call last):
...
  File "/usr/lib/python3.9/site-packages/visidata/loaders/http.py", line 50, in _iter_lines
    links = urllib.parse.parse_header(linkhdr)
AttributeError: module 'urllib.parse' has no attribute 'parse_header'

```

Reproduce: 
```
vd https://blog.nature.org/2021/04/12/beaver-otter-muskrat-a-field-guide-to-freshwater-mammals/
```

Please attach the commandlog (saved with `Ctrl-D`) to show the steps that led to the issue.
See [here](http://visidata.org/docs/save-restore/) for more details.

**Additional context**
Please include the version of VisiData. Latest develop version of VisData.  Python 3.9.2


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `visidata/loaders/http.py`
Content:
```
1 from visidata import Path, RepeatFile, vd, VisiData
2 from visidata.loaders.tsv import splitter
3 
4 content_filetypes = {
5     'tab-separated-values': 'tsv'
6 }
7 
8 vd.option('http_max_next', 0, 'max next.url pages to follow in http response') #848
9 vd.option('http_req_headers', {}, 'http headers to send to requests')
10 
11 
12 @VisiData.api
13 def openurl_http(vd, path, filetype=None):
14     schemes = path.scheme.split('+')
15     if len(schemes) > 1:
16         sch = schemes[0]
17         openfunc = getattr(vd, f'openhttp_{sch}', vd.getGlobals().get(f'openhttp_{sch}'))
18         if not openfunc:
19             vd.fail(f'no vd.openhttp_{sch}')
20         return openfunc(Path(schemes[-1]+'://'+path.given.split('://')[1]))
21 
22     import urllib.request
23     import urllib.error
24     import mimetypes
25 
26     # fallback to mime-type
27     req = urllib.request.Request(path.given, **vd.options.getall('http_req_'))
28     response = urllib.request.urlopen(req)
29 
30     contenttype = response.getheader('content-type')
31     subtype = contenttype.split(';')[0].split('/')[-1]
32 
33     filetype = filetype or vd.guessFiletype(path, funcprefix='guessurl_').get('filetype')
34     filetype = filetype or content_filetypes.get(subtype, subtype)
35     filetype = filetype or vd.guessFiletype(path).get('filetype')
36 
37     # Automatically paginate if a 'next' URL is given
38     def _iter_lines(path=path, response=response, max_next=vd.options.http_max_next):
39         path.responses = []
40         n = 0
41         while response:
42             path.responses.append(response)
43             with response as fp:
44                 for line in splitter(response, delim=b'\n'):
45                     yield line.decode(vd.options.encoding)
46 
47             linkhdr = response.getheader('Link')
48             src = None
49             if linkhdr:
50                 links = urllib.parse.parse_header(linkhdr)
51                 src = links.get('next', {}).get('url', None)
52 
53             if not src:
54                 break
55 
56             n += 1
57             if n > max_next:
58                 vd.warning(f'stopping at max {max_next} pages')
59                 break
60 
61             vd.status(f'fetching next page from {src}')
62             response = requests.get(src, stream=True, **vd.options.getall('http_req_'))
63 
64     # add resettable iterator over contents as an already-open fp
65     path.fptext = RepeatFile(_iter_lines())
66 
67     return vd.openSource(path, filetype=filetype)
68 
69 
70 VisiData.openurl_https = VisiData.openurl_http
71
```

Path: `visidata/plugins.py`
Content:
```
1 import os.path
2 import os
3 import sys
4 import re
5 import shutil
6 import importlib
7 import subprocess
8 import urllib
9 
10 from visidata import VisiData, vd, Path, CellColorizer, JsonLinesSheet, AttrDict, Column, Progress, ExpectedException, BaseSheet, asyncsingle, asyncthread
11 
12 
13 vd.option('plugins_url', 'https://visidata.org/plugins/plugins.jsonl', 'source of plugins sheet')
14 vd.option('plugins_autoload', True, 'do not autoload plugins if False')
15 
16 
17 @VisiData.lazy_property
18 def pluginsSheet(p):
19     return PluginsSheet('plugins_global')
20 
21 def _plugin_path(plugin):
22     return Path(os.path.join(vd.options.visidata_dir, "plugins", plugin.name+".py"))
23 
24 def _plugin_init():
25     return Path(os.path.join(vd.options.visidata_dir, "plugins", "__init__.py"))
26 
27 def _plugin_import(plugin):
28     return "import " + _plugin_import_name(plugin)
29 
30 def _plugin_import_name(plugin):
31     if not plugin.url:
32         return 'visidata.plugins.'+plugin.name
33     if 'git+' in plugin.url:
34         return plugin.name
35     return "plugins."+plugin.name
36 
37 def _plugin_in_import_list(plugin):
38     with Path(_plugin_init()).open(mode='r', encoding='utf-8') as fprc:
39         r = re.compile(r'^{}\W'.format(_plugin_import(plugin)))
40         for line in fprc.readlines():
41             if r.match(line):
42                 return True
43 
44 def _installedStatus(col, plugin):
45     return '*' if importlib.util.find_spec(_plugin_import_name(plugin)) else ''
46 
47 def _loadedVersion(plugin):
48     name = _plugin_import_name(plugin)
49     if name not in sys.modules:
50         return ''
51     mod = sys.modules[name]
52     return getattr(mod, '__version__', 'unknown version installed')
53 
54 def _checkHash(data, sha):
55     import hashlib
56     return hashlib.sha256(data.strip().encode('utf-8')).hexdigest() == sha
57 
58 def _pluginColorizer(s,c,r,v):
59     if not r: return None
60     ver = _loadedVersion(r)
61     if not ver: return None
62     if not r.latest_ver: return None
63     if ver != r.latest_ver: return 'color_warning'
64     return 'color_working'
65 
66 
67 @VisiData.api
68 def pipinstall(vd, deps):
69     'Install *deps*, a list of pypi modules to install via pip into the plugins-deps directory. Return True if successful (no error).'
70     p = subprocess.Popen([sys.executable, '-m', 'pip', 'install',
71                         '--target', str(Path(vd.options.visidata_dir)/"plugins-deps"),
72                       ] + deps,
73             stdout=subprocess.PIPE,
74             stderr=subprocess.PIPE)
75     out, err = p.communicate()
76     vd.status(out.decode())
77     if err:
78         vd.warning(err.decode())
79         return False
80     return True
81 
82 
83 class PluginsSheet(JsonLinesSheet):
84     rowtype = "plugins"  # rowdef: AttrDict of json dict
85     colorizers = [
86         CellColorizer(3, None, _pluginColorizer)
87     ]
88 
89     def iterload(self):
90         for r in JsonLinesSheet.iterload(self):
91             yield AttrDict(r)
92 
93     @asyncsingle
94     def reload(self):
95         try:
96             self.source = vd.urlcache(vd.options.plugins_url or vd.fail(), days=1)  # for VisiDataMetaSheet.reload()
97         except urllib.error.URLError as e:
98             vd.debug(e)
99             return
100 
101         super().reload.__wrapped__(self)
102         self.addColumn(Column('available', width=0, getter=_installedStatus), index=1)
103         self.addColumn(Column('installed', width=8, getter=lambda c,r: _loadedVersion(r)), index=2)
104         self.column('description').width = 40
105         self.setKeys([self.column("name")])
106 
107         for name, mod in sys.modules.items():
108             if name.startswith('visidata.plugins.'):
109                 self.addRow(AttrDict(name='.'.join(name.split('.')[2:]),
110                                  description=getattr(mod, '__description__', mod.__doc__),
111                                  maintainer=getattr(mod, '__author__', None),
112                                  latest_release='',
113                                  latest_ver='',
114                                  url=''
115                                  ))
116 
117         for r in Progress(self.rows):
118             for funcname in (r.provides or '').split():
119                 func = lambda *args, **kwargs: vd.fail('this requires the %s plugin' % r.name)
120                 vd.addGlobals({funcname: func})
121                 setattr(vd, funcname, func)
122 
123         # check for plugins with newer versions
124         def is_stale(r):
125             v = _loadedVersion(r)
126             return v and r.latest_ver and v != r.latest_ver
127 
128         stale_plugins = list(filter(is_stale, self.rows))
129         if len(stale_plugins) > 0:
130             vd.warning(f'update available for {len(stale_plugins)} plugins')
131 
132     def installPlugin(self, plugin):
133         # pip3 install requirements
134         initpath = _plugin_init()
135         os.makedirs(initpath.parent, exist_ok=True)
136         if not initpath.exists():
137             initpath.touch()
138 
139         outpath = _plugin_path(plugin)
140         overwrite = True
141         if outpath.exists():
142             try:
143                 vd.confirm("plugin path already exists, overwrite? ")
144             except ExpectedException:
145                 overwrite = False
146                 if _plugin_in_import_list(plugin):
147                     vd.fail("plugin already loaded")
148                 else:
149                     self._loadPlugin(plugin)
150         if overwrite:
151             self._install(plugin)
152 
153     @asyncthread
154     def _install(self, plugin):
155         outpath = _plugin_path(plugin)
156 
157         if "git+" in plugin.url:
158             p = subprocess.Popen([sys.executable, '-m', 'pip', 'install',
159                                 '--upgrade', plugin.url],
160                     stdout=subprocess.PIPE,
161                     stderr=subprocess.PIPE)
162             out, err = p.communicate()
163             vd.status(out.decode())
164             if err:
165                 vd.warning(err.decode())
166             if p.returncode != 0:
167                 vd.fail('pip install failed')
168         else:
169             with vd.urlcache(plugin.url, days=0).open(encoding='utf-8') as pyfp:
170                 contents = pyfp.read()
171                 if plugin.sha256:
172                     if not _checkHash(contents, plugin.sha256):
173                         vd.error('%s plugin SHA256 does not match!' % plugin.name)
174                 else:
175                     vd.warning('no SHA256 provided for %s plugin, not validating' % plugin.name)
176                 with outpath.open(mode='w', encoding='utf-8') as outfp:
177                     outfp.write(contents)
178 
179         if plugin.pydeps:
180             vd.pipinstall(plugin.pydeps.split())
181 
182         vd.status('%s plugin installed' % plugin.name)
183 
184         if _plugin_in_import_list(plugin):
185             vd.warning("plugin already loaded")
186         else:
187             self._loadPlugin(plugin)
188 
189 
190     def _loadPlugin(self, plugin):
191         with Path(_plugin_init()).open(mode='a', encoding='utf-8') as fprc:
192             print(_plugin_import(plugin), file=fprc)
193             importlib.import_module(_plugin_import_name(plugin))
194             vd.status('%s plugin loaded' % plugin.name)
195 
196 
197     def removePluginIfExists(self, plugin):
198         self.removePlugin(plugin)
199 
200     def removePlugin(self, plugin):
201         if not _plugin_in_import_list(plugin):
202             vd.fail("plugin not in import list")
203 
204         initpath = Path(_plugin_init())
205         oldinitpath = Path(initpath.with_suffix(initpath.suffix + '.bak'))
206         try:
207             shutil.copyfile(initpath, oldinitpath)
208 
209             # Copy lines from the backup init file into its replacement, skipping lines that import the removed plugin.
210             #
211             # By matching from the start of a line through a word boundary, we avoid removing commented lines or inadvertently removing
212             # plugins with similar names.
213             with oldinitpath.open(encoding='utf-8') as old, initpath.open(mode='w', encoding='utf-8') as new:
214                 r = re.compile(r'^{}\W'.format(_plugin_import(plugin)))
215                 new.writelines(line for line in old.readlines() if not r.match(line))
216 
217             if os.path.exists(_plugin_path(plugin)):
218                 os.unlink(_plugin_path(plugin))
219             sys.modules.pop(_plugin_import_name(plugin))
220             importlib.invalidate_caches()
221             vd.warning('{0} plugin uninstalled'.format(plugin['name']))
222         except FileNotFoundError:
223             vd.warning("no plugins/__init__.py found")
224 
225 
226 BaseSheet.addCommand(None, 'open-plugins', 'vd.push(vd.pluginsSheet)', 'Open Plugins Sheet to manage supported plugins')
227 
228 PluginsSheet.addCommand('a', 'add-plugin', 'installPlugin(cursorRow)', 'Install and enable current plugin')
229 PluginsSheet.addCommand('d', 'delete-plugin', 'removePluginIfExists(cursorRow)', 'Disable current plugin')
230 
231 vd.addMenuItems('''
232     View > Plugins > open-plugins
233 ''')
234
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/visidata/loaders/http.py b/visidata/loaders/http.py
--- a/visidata/loaders/http.py
+++ b/visidata/loaders/http.py
@@ -1,3 +1,5 @@
+import re
+
 from visidata import Path, RepeatFile, vd, VisiData
 from visidata.loaders.tsv import splitter
 
@@ -47,24 +49,59 @@
             linkhdr = response.getheader('Link')
             src = None
             if linkhdr:
-                links = urllib.parse.parse_header(linkhdr)
-                src = links.get('next', {}).get('url', None)
+                links = parse_header_links(linkhdr)
+                link_data = {}
+                for link in links:
+                    key = link.get('rel') or link.get('url')
+                    link_data[key] = link
+                src = link_data.get('next', {}).get('url', None)
 
             if not src:
                 break
 
             n += 1
             if n > max_next:
-                vd.warning(f'stopping at max {max_next} pages')
+                vd.warning(f'stopping at max next pages: {max_next} pages')
                 break
 
             vd.status(f'fetching next page from {src}')
-            response = requests.get(src, stream=True, **vd.options.getall('http_req_'))
+            req = urllib.request.Request(src, **vd.options.getall('http_req_'))
+            response = urllib.request.urlopen(req)
 
     # add resettable iterator over contents as an already-open fp
     path.fptext = RepeatFile(_iter_lines())
 
     return vd.openSource(path, filetype=filetype)
 
+def parse_header_links(link_header):
+    '''Return a list of dictionaries:
+    [{'url': 'https://example.com/content?page=1', 'rel': 'prev'},
+     {'url': 'https://example.com/content?page=3', 'rel': 'next'}]
+    Takes a link header string, of the form
+    '<https://example.com/content?page=1>; rel="prev", <https://example.com/content?page=3>; rel="next"'
+    See https://datatracker.ietf.org/doc/html/rfc8288#section-3
+    '''
+
+    links = []
+    quote_space = ' \'"'
+    link_header = link_header.strip(quote_space)
+    if not link_header: return []
+    for link_value in re.split(', *<', link_header):
+        if ';' in link_value:
+            url, params = link_value.split(';', maxsplit=1)
+        else:
+            url, params = link_value, ''
+        link = {'url': url.strip('<>' + quote_space)}
+
+        for param in params.split(';'):
+            if '=' in param:
+                key, value = param.split('=')
+                key = key.strip(quote_space)
+                value = value.strip(quote_space)
+                link[key] = value
+            else:
+                break
+        links.append(link)
+    return links
 
 VisiData.openurl_https = VisiData.openurl_http
diff --git a/visidata/plugins.py b/visidata/plugins.py
--- a/visidata/plugins.py
+++ b/visidata/plugins.py
@@ -5,7 +5,7 @@
 import shutil
 import importlib
 import subprocess
-import urllib
+import urllib.error
 
 from visidata import VisiData, vd, Path, CellColorizer, JsonLinesSheet, AttrDict, Column, Progress, ExpectedException, BaseSheet, asyncsingle, asyncthread

{"golden_diff": "diff --git a/visidata/loaders/http.py b/visidata/loaders/http.py\n--- a/visidata/loaders/http.py\n+++ b/visidata/loaders/http.py\n@@ -1,3 +1,5 @@\n+import re\n+\n from visidata import Path, RepeatFile, vd, VisiData\n from visidata.loaders.tsv import splitter\n \n@@ -47,24 +49,59 @@\n             linkhdr = response.getheader('Link')\n             src = None\n             if linkhdr:\n-                links = urllib.parse.parse_header(linkhdr)\n-                src = links.get('next', {}).get('url', None)\n+                links = parse_header_links(linkhdr)\n+                link_data = {}\n+                for link in links:\n+                    key = link.get('rel') or link.get('url')\n+                    link_data[key] = link\n+                src = link_data.get('next', {}).get('url', None)\n \n             if not src:\n                 break\n \n             n += 1\n             if n > max_next:\n-                vd.warning(f'stopping at max {max_next} pages')\n+                vd.warning(f'stopping at max next pages: {max_next} pages')\n                 break\n \n             vd.status(f'fetching next page from {src}')\n-            response = requests.get(src, stream=True, **vd.options.getall('http_req_'))\n+            req = urllib.request.Request(src, **vd.options.getall('http_req_'))\n+            response = urllib.request.urlopen(req)\n \n     # add resettable iterator over contents as an already-open fp\n     path.fptext = RepeatFile(_iter_lines())\n \n     return vd.openSource(path, filetype=filetype)\n \n+def parse_header_links(link_header):\n+    '''Return a list of dictionaries:\n+    [{'url': 'https://example.com/content?page=1', 'rel': 'prev'},\n+     {'url': 'https://example.com/content?page=3', 'rel': 'next'}]\n+    Takes a link header string, of the form\n+    '<https://example.com/content?page=1>; rel=\"prev\", <https://example.com/content?page=3>; rel=\"next\"'\n+    See https://datatracker.ietf.org/doc/html/rfc8288#section-3\n+    '''\n+\n+    links = []\n+    quote_space = ' \\'\"'\n+    link_header = link_header.strip(quote_space)\n+    if not link_header: return []\n+    for link_value in re.split(', *<', link_header):\n+        if ';' in link_value:\n+            url, params = link_value.split(';', maxsplit=1)\n+        else:\n+            url, params = link_value, ''\n+        link = {'url': url.strip('<>' + quote_space)}\n+\n+        for param in params.split(';'):\n+            if '=' in param:\n+                key, value = param.split('=')\n+                key = key.strip(quote_space)\n+                value = value.strip(quote_space)\n+                link[key] = value\n+            else:\n+                break\n+        links.append(link)\n+    return links\n \n VisiData.openurl_https = VisiData.openurl_http\ndiff --git a/visidata/plugins.py b/visidata/plugins.py\n--- a/visidata/plugins.py\n+++ b/visidata/plugins.py\n@@ -5,7 +5,7 @@\n import shutil\n import importlib\n import subprocess\n-import urllib\n+import urllib.error\n \n from visidata import VisiData, vd, Path, CellColorizer, JsonLinesSheet, AttrDict, Column, Progress, ExpectedException, BaseSheet, asyncsingle, asyncthread\n", "issue": "[urllib imports]  Multiple urllib imports are missing\n**Small description**\r\nMissing imports for `urllib.error` and `urllib.parse`\r\n\r\n\r\n**Expected result**\r\nNo stack traces from missing imports.\r\n\r\n\r\n**Actual result with screenshot**\r\n\r\nFirst Trace\r\n```\r\nTraceback (most recent call last):\r\n  File \"/usr/lib/python3.9/site-packages/visidata/threads.py\", line 200, in _toplevelTryFunc\r\n    t.status = func(*args, **kwargs)\r\n  File \"/usr/lib/python3.9/site-packages/visidata/threads.py\", line 219, in _func\r\n    func(*args, **kwargs)\r\n  File \"/usr/lib/python3.9/site-packages/visidata/plugins.py\", line 97, in reload\r\n    except urllib.error.URLError as e:\r\nAttributeError: module 'urllib' has no attribute 'error'\r\n```\r\n\r\nSecond trace\r\n```\r\nTraceback (most recent call last):\r\n...\r\n  File \"/usr/lib/python3.9/site-packages/visidata/loaders/http.py\", line 50, in _iter_lines\r\n    links = urllib.parse.parse_header(linkhdr)\r\nAttributeError: module 'urllib.parse' has no attribute 'parse_header'\r\n\r\n```\r\n\r\nReproduce: \r\n```\r\nvd https://blog.nature.org/2021/04/12/beaver-otter-muskrat-a-field-guide-to-freshwater-mammals/\r\n```\r\n\r\nPlease attach the commandlog (saved with `Ctrl-D`) to show the steps that led to the issue.\r\nSee [here](http://visidata.org/docs/save-restore/) for more details.\r\n\r\n**Additional context**\r\nPlease include the version of VisiData. Latest develop version of VisData.  Python 3.9.2\r\n\n", "before_files": [{"content": "from visidata import Path, RepeatFile, vd, VisiData\nfrom visidata.loaders.tsv import splitter\n\ncontent_filetypes = {\n    'tab-separated-values': 'tsv'\n}\n\nvd.option('http_max_next', 0, 'max next.url pages to follow in http response') #848\nvd.option('http_req_headers', {}, 'http headers to send to requests')\n\n\n@VisiData.api\ndef openurl_http(vd, path, filetype=None):\n    schemes = path.scheme.split('+')\n    if len(schemes) > 1:\n        sch = schemes[0]\n        openfunc = getattr(vd, f'openhttp_{sch}', vd.getGlobals().get(f'openhttp_{sch}'))\n        if not openfunc:\n            vd.fail(f'no vd.openhttp_{sch}')\n        return openfunc(Path(schemes[-1]+'://'+path.given.split('://')[1]))\n\n    import urllib.request\n    import urllib.error\n    import mimetypes\n\n    # fallback to mime-type\n    req = urllib.request.Request(path.given, **vd.options.getall('http_req_'))\n    response = urllib.request.urlopen(req)\n\n    contenttype = response.getheader('content-type')\n    subtype = contenttype.split(';')[0].split('/')[-1]\n\n    filetype = filetype or vd.guessFiletype(path, funcprefix='guessurl_').get('filetype')\n    filetype = filetype or content_filetypes.get(subtype, subtype)\n    filetype = filetype or vd.guessFiletype(path).get('filetype')\n\n    # Automatically paginate if a 'next' URL is given\n    def _iter_lines(path=path, response=response, max_next=vd.options.http_max_next):\n        path.responses = []\n        n = 0\n        while response:\n            path.responses.append(response)\n            with response as fp:\n                for line in splitter(response, delim=b'\\n'):\n                    yield line.decode(vd.options.encoding)\n\n            linkhdr = response.getheader('Link')\n            src = None\n            if linkhdr:\n                links = urllib.parse.parse_header(linkhdr)\n                src = links.get('next', {}).get('url', None)\n\n            if not src:\n                break\n\n            n += 1\n            if n > max_next:\n                vd.warning(f'stopping at max {max_next} pages')\n                break\n\n            vd.status(f'fetching next page from {src}')\n            response = requests.get(src, stream=True, **vd.options.getall('http_req_'))\n\n    # add resettable iterator over contents as an already-open fp\n    path.fptext = RepeatFile(_iter_lines())\n\n    return vd.openSource(path, filetype=filetype)\n\n\nVisiData.openurl_https = VisiData.openurl_http\n", "path": "visidata/loaders/http.py"}, {"content": "import os.path\nimport os\nimport sys\nimport re\nimport shutil\nimport importlib\nimport subprocess\nimport urllib\n\nfrom visidata import VisiData, vd, Path, CellColorizer, JsonLinesSheet, AttrDict, Column, Progress, ExpectedException, BaseSheet, asyncsingle, asyncthread\n\n\nvd.option('plugins_url', 'https://visidata.org/plugins/plugins.jsonl', 'source of plugins sheet')\nvd.option('plugins_autoload', True, 'do not autoload plugins if False')\n\n\n@VisiData.lazy_property\ndef pluginsSheet(p):\n    return PluginsSheet('plugins_global')\n\ndef _plugin_path(plugin):\n    return Path(os.path.join(vd.options.visidata_dir, \"plugins\", plugin.name+\".py\"))\n\ndef _plugin_init():\n    return Path(os.path.join(vd.options.visidata_dir, \"plugins\", \"__init__.py\"))\n\ndef _plugin_import(plugin):\n    return \"import \" + _plugin_import_name(plugin)\n\ndef _plugin_import_name(plugin):\n    if not plugin.url:\n        return 'visidata.plugins.'+plugin.name\n    if 'git+' in plugin.url:\n        return plugin.name\n    return \"plugins.\"+plugin.name\n\ndef _plugin_in_import_list(plugin):\n    with Path(_plugin_init()).open(mode='r', encoding='utf-8') as fprc:\n        r = re.compile(r'^{}\\W'.format(_plugin_import(plugin)))\n        for line in fprc.readlines():\n            if r.match(line):\n                return True\n\ndef _installedStatus(col, plugin):\n    return '*' if importlib.util.find_spec(_plugin_import_name(plugin)) else ''\n\ndef _loadedVersion(plugin):\n    name = _plugin_import_name(plugin)\n    if name not in sys.modules:\n        return ''\n    mod = sys.modules[name]\n    return getattr(mod, '__version__', 'unknown version installed')\n\ndef _checkHash(data, sha):\n    import hashlib\n    return hashlib.sha256(data.strip().encode('utf-8')).hexdigest() == sha\n\ndef _pluginColorizer(s,c,r,v):\n    if not r: return None\n    ver = _loadedVersion(r)\n    if not ver: return None\n    if not r.latest_ver: return None\n    if ver != r.latest_ver: return 'color_warning'\n    return 'color_working'\n\n\n@VisiData.api\ndef pipinstall(vd, deps):\n    'Install *deps*, a list of pypi modules to install via pip into the plugins-deps directory. Return True if successful (no error).'\n    p = subprocess.Popen([sys.executable, '-m', 'pip', 'install',\n                        '--target', str(Path(vd.options.visidata_dir)/\"plugins-deps\"),\n                      ] + deps,\n            stdout=subprocess.PIPE,\n            stderr=subprocess.PIPE)\n    out, err = p.communicate()\n    vd.status(out.decode())\n    if err:\n        vd.warning(err.decode())\n        return False\n    return True\n\n\nclass PluginsSheet(JsonLinesSheet):\n    rowtype = \"plugins\"  # rowdef: AttrDict of json dict\n    colorizers = [\n        CellColorizer(3, None, _pluginColorizer)\n    ]\n\n    def iterload(self):\n        for r in JsonLinesSheet.iterload(self):\n            yield AttrDict(r)\n\n    @asyncsingle\n    def reload(self):\n        try:\n            self.source = vd.urlcache(vd.options.plugins_url or vd.fail(), days=1)  # for VisiDataMetaSheet.reload()\n        except urllib.error.URLError as e:\n            vd.debug(e)\n            return\n\n        super().reload.__wrapped__(self)\n        self.addColumn(Column('available', width=0, getter=_installedStatus), index=1)\n        self.addColumn(Column('installed', width=8, getter=lambda c,r: _loadedVersion(r)), index=2)\n        self.column('description').width = 40\n        self.setKeys([self.column(\"name\")])\n\n        for name, mod in sys.modules.items():\n            if name.startswith('visidata.plugins.'):\n                self.addRow(AttrDict(name='.'.join(name.split('.')[2:]),\n                                 description=getattr(mod, '__description__', mod.__doc__),\n                                 maintainer=getattr(mod, '__author__', None),\n                                 latest_release='',\n                                 latest_ver='',\n                                 url=''\n                                 ))\n\n        for r in Progress(self.rows):\n            for funcname in (r.provides or '').split():\n                func = lambda *args, **kwargs: vd.fail('this requires the %s plugin' % r.name)\n                vd.addGlobals({funcname: func})\n                setattr(vd, funcname, func)\n\n        # check for plugins with newer versions\n        def is_stale(r):\n            v = _loadedVersion(r)\n            return v and r.latest_ver and v != r.latest_ver\n\n        stale_plugins = list(filter(is_stale, self.rows))\n        if len(stale_plugins) > 0:\n            vd.warning(f'update available for {len(stale_plugins)} plugins')\n\n    def installPlugin(self, plugin):\n        # pip3 install requirements\n        initpath = _plugin_init()\n        os.makedirs(initpath.parent, exist_ok=True)\n        if not initpath.exists():\n            initpath.touch()\n\n        outpath = _plugin_path(plugin)\n        overwrite = True\n        if outpath.exists():\n            try:\n                vd.confirm(\"plugin path already exists, overwrite? \")\n            except ExpectedException:\n                overwrite = False\n                if _plugin_in_import_list(plugin):\n                    vd.fail(\"plugin already loaded\")\n                else:\n                    self._loadPlugin(plugin)\n        if overwrite:\n            self._install(plugin)\n\n    @asyncthread\n    def _install(self, plugin):\n        outpath = _plugin_path(plugin)\n\n        if \"git+\" in plugin.url:\n            p = subprocess.Popen([sys.executable, '-m', 'pip', 'install',\n                                '--upgrade', plugin.url],\n                    stdout=subprocess.PIPE,\n                    stderr=subprocess.PIPE)\n            out, err = p.communicate()\n            vd.status(out.decode())\n            if err:\n                vd.warning(err.decode())\n            if p.returncode != 0:\n                vd.fail('pip install failed')\n        else:\n            with vd.urlcache(plugin.url, days=0).open(encoding='utf-8') as pyfp:\n                contents = pyfp.read()\n                if plugin.sha256:\n                    if not _checkHash(contents, plugin.sha256):\n                        vd.error('%s plugin SHA256 does not match!' % plugin.name)\n                else:\n                    vd.warning('no SHA256 provided for %s plugin, not validating' % plugin.name)\n                with outpath.open(mode='w', encoding='utf-8') as outfp:\n                    outfp.write(contents)\n\n        if plugin.pydeps:\n            vd.pipinstall(plugin.pydeps.split())\n\n        vd.status('%s plugin installed' % plugin.name)\n\n        if _plugin_in_import_list(plugin):\n            vd.warning(\"plugin already loaded\")\n        else:\n            self._loadPlugin(plugin)\n\n\n    def _loadPlugin(self, plugin):\n        with Path(_plugin_init()).open(mode='a', encoding='utf-8') as fprc:\n            print(_plugin_import(plugin), file=fprc)\n            importlib.import_module(_plugin_import_name(plugin))\n            vd.status('%s plugin loaded' % plugin.name)\n\n\n    def removePluginIfExists(self, plugin):\n        self.removePlugin(plugin)\n\n    def removePlugin(self, plugin):\n        if not _plugin_in_import_list(plugin):\n            vd.fail(\"plugin not in import list\")\n\n        initpath = Path(_plugin_init())\n        oldinitpath = Path(initpath.with_suffix(initpath.suffix + '.bak'))\n        try:\n            shutil.copyfile(initpath, oldinitpath)\n\n            # Copy lines from the backup init file into its replacement, skipping lines that import the removed plugin.\n            #\n            # By matching from the start of a line through a word boundary, we avoid removing commented lines or inadvertently removing\n            # plugins with similar names.\n            with oldinitpath.open(encoding='utf-8') as old, initpath.open(mode='w', encoding='utf-8') as new:\n                r = re.compile(r'^{}\\W'.format(_plugin_import(plugin)))\n                new.writelines(line for line in old.readlines() if not r.match(line))\n\n            if os.path.exists(_plugin_path(plugin)):\n                os.unlink(_plugin_path(plugin))\n            sys.modules.pop(_plugin_import_name(plugin))\n            importlib.invalidate_caches()\n            vd.warning('{0} plugin uninstalled'.format(plugin['name']))\n        except FileNotFoundError:\n            vd.warning(\"no plugins/__init__.py found\")\n\n\nBaseSheet.addCommand(None, 'open-plugins', 'vd.push(vd.pluginsSheet)', 'Open Plugins Sheet to manage supported plugins')\n\nPluginsSheet.addCommand('a', 'add-plugin', 'installPlugin(cursorRow)', 'Install and enable current plugin')\nPluginsSheet.addCommand('d', 'delete-plugin', 'removePluginIfExists(cursorRow)', 'Disable current plugin')\n\nvd.addMenuItems('''\n    View > Plugins > open-plugins\n''')\n", "path": "visidata/plugins.py"}], "after_files": [{"content": "import re\n\nfrom visidata import Path, RepeatFile, vd, VisiData\nfrom visidata.loaders.tsv import splitter\n\ncontent_filetypes = {\n    'tab-separated-values': 'tsv'\n}\n\nvd.option('http_max_next', 0, 'max next.url pages to follow in http response') #848\nvd.option('http_req_headers', {}, 'http headers to send to requests')\n\n\n@VisiData.api\ndef openurl_http(vd, path, filetype=None):\n    schemes = path.scheme.split('+')\n    if len(schemes) > 1:\n        sch = schemes[0]\n        openfunc = getattr(vd, f'openhttp_{sch}', vd.getGlobals().get(f'openhttp_{sch}'))\n        if not openfunc:\n            vd.fail(f'no vd.openhttp_{sch}')\n        return openfunc(Path(schemes[-1]+'://'+path.given.split('://')[1]))\n\n    import urllib.request\n    import urllib.error\n    import mimetypes\n\n    # fallback to mime-type\n    req = urllib.request.Request(path.given, **vd.options.getall('http_req_'))\n    response = urllib.request.urlopen(req)\n\n    contenttype = response.getheader('content-type')\n    subtype = contenttype.split(';')[0].split('/')[-1]\n\n    filetype = filetype or vd.guessFiletype(path, funcprefix='guessurl_').get('filetype')\n    filetype = filetype or content_filetypes.get(subtype, subtype)\n    filetype = filetype or vd.guessFiletype(path).get('filetype')\n\n    # Automatically paginate if a 'next' URL is given\n    def _iter_lines(path=path, response=response, max_next=vd.options.http_max_next):\n        path.responses = []\n        n = 0\n        while response:\n            path.responses.append(response)\n            with response as fp:\n                for line in splitter(response, delim=b'\\n'):\n                    yield line.decode(vd.options.encoding)\n\n            linkhdr = response.getheader('Link')\n            src = None\n            if linkhdr:\n                links = parse_header_links(linkhdr)\n                link_data = {}\n                for link in links:\n                    key = link.get('rel') or link.get('url')\n                    link_data[key] = link\n                src = link_data.get('next', {}).get('url', None)\n\n            if not src:\n                break\n\n            n += 1\n            if n > max_next:\n                vd.warning(f'stopping at max next pages: {max_next} pages')\n                break\n\n            vd.status(f'fetching next page from {src}')\n            req = urllib.request.Request(src, **vd.options.getall('http_req_'))\n            response = urllib.request.urlopen(req)\n\n    # add resettable iterator over contents as an already-open fp\n    path.fptext = RepeatFile(_iter_lines())\n\n    return vd.openSource(path, filetype=filetype)\n\ndef parse_header_links(link_header):\n    '''Return a list of dictionaries:\n    [{'url': 'https://example.com/content?page=1', 'rel': 'prev'},\n     {'url': 'https://example.com/content?page=3', 'rel': 'next'}]\n    Takes a link header string, of the form\n    '<https://example.com/content?page=1>; rel=\"prev\", <https://example.com/content?page=3>; rel=\"next\"'\n    See https://datatracker.ietf.org/doc/html/rfc8288#section-3\n    '''\n\n    links = []\n    quote_space = ' \\'\"'\n    link_header = link_header.strip(quote_space)\n    if not link_header: return []\n    for link_value in re.split(', *<', link_header):\n        if ';' in link_value:\n            url, params = link_value.split(';', maxsplit=1)\n        else:\n            url, params = link_value, ''\n        link = {'url': url.strip('<>' + quote_space)}\n\n        for param in params.split(';'):\n            if '=' in param:\n                key, value = param.split('=')\n                key = key.strip(quote_space)\n                value = value.strip(quote_space)\n                link[key] = value\n            else:\n                break\n        links.append(link)\n    return links\n\nVisiData.openurl_https = VisiData.openurl_http\n", "path": "visidata/loaders/http.py"}, {"content": "import os.path\nimport os\nimport sys\nimport re\nimport shutil\nimport importlib\nimport subprocess\nimport urllib.error\n\nfrom visidata import VisiData, vd, Path, CellColorizer, JsonLinesSheet, AttrDict, Column, Progress, ExpectedException, BaseSheet, asyncsingle, asyncthread\n\n\nvd.option('plugins_url', 'https://visidata.org/plugins/plugins.jsonl', 'source of plugins sheet')\nvd.option('plugins_autoload', True, 'do not autoload plugins if False')\n\n\n@VisiData.lazy_property\ndef pluginsSheet(p):\n    return PluginsSheet('plugins_global')\n\ndef _plugin_path(plugin):\n    return Path(os.path.join(vd.options.visidata_dir, \"plugins\", plugin.name+\".py\"))\n\ndef _plugin_init():\n    return Path(os.path.join(vd.options.visidata_dir, \"plugins\", \"__init__.py\"))\n\ndef _plugin_import(plugin):\n    return \"import \" + _plugin_import_name(plugin)\n\ndef _plugin_import_name(plugin):\n    if not plugin.url:\n        return 'visidata.plugins.'+plugin.name\n    if 'git+' in plugin.url:\n        return plugin.name\n    return \"plugins.\"+plugin.name\n\ndef _plugin_in_import_list(plugin):\n    with Path(_plugin_init()).open(mode='r', encoding='utf-8') as fprc:\n        r = re.compile(r'^{}\\W'.format(_plugin_import(plugin)))\n        for line in fprc.readlines():\n            if r.match(line):\n                return True\n\ndef _installedStatus(col, plugin):\n    return '*' if importlib.util.find_spec(_plugin_import_name(plugin)) else ''\n\ndef _loadedVersion(plugin):\n    name = _plugin_import_name(plugin)\n    if name not in sys.modules:\n        return ''\n    mod = sys.modules[name]\n    return getattr(mod, '__version__', 'unknown version installed')\n\ndef _checkHash(data, sha):\n    import hashlib\n    return hashlib.sha256(data.strip().encode('utf-8')).hexdigest() == sha\n\ndef _pluginColorizer(s,c,r,v):\n    if not r: return None\n    ver = _loadedVersion(r)\n    if not ver: return None\n    if not r.latest_ver: return None\n    if ver != r.latest_ver: return 'color_warning'\n    return 'color_working'\n\n\n@VisiData.api\ndef pipinstall(vd, deps):\n    'Install *deps*, a list of pypi modules to install via pip into the plugins-deps directory. Return True if successful (no error).'\n    p = subprocess.Popen([sys.executable, '-m', 'pip', 'install',\n                        '--target', str(Path(vd.options.visidata_dir)/\"plugins-deps\"),\n                      ] + deps,\n            stdout=subprocess.PIPE,\n            stderr=subprocess.PIPE)\n    out, err = p.communicate()\n    vd.status(out.decode())\n    if err:\n        vd.warning(err.decode())\n        return False\n    return True\n\n\nclass PluginsSheet(JsonLinesSheet):\n    rowtype = \"plugins\"  # rowdef: AttrDict of json dict\n    colorizers = [\n        CellColorizer(3, None, _pluginColorizer)\n    ]\n\n    def iterload(self):\n        for r in JsonLinesSheet.iterload(self):\n            yield AttrDict(r)\n\n    @asyncsingle\n    def reload(self):\n        try:\n            self.source = vd.urlcache(vd.options.plugins_url or vd.fail(), days=1)  # for VisiDataMetaSheet.reload()\n        except urllib.error.URLError as e:\n            vd.debug(e)\n            return\n\n        super().reload.__wrapped__(self)\n        self.addColumn(Column('available', width=0, getter=_installedStatus), index=1)\n        self.addColumn(Column('installed', width=8, getter=lambda c,r: _loadedVersion(r)), index=2)\n        self.column('description').width = 40\n        self.setKeys([self.column(\"name\")])\n\n        for name, mod in sys.modules.items():\n            if name.startswith('visidata.plugins.'):\n                self.addRow(AttrDict(name='.'.join(name.split('.')[2:]),\n                                 description=getattr(mod, '__description__', mod.__doc__),\n                                 maintainer=getattr(mod, '__author__', None),\n                                 latest_release='',\n                                 latest_ver='',\n                                 url=''\n                                 ))\n\n        for r in Progress(self.rows):\n            for funcname in (r.provides or '').split():\n                func = lambda *args, **kwargs: vd.fail('this requires the %s plugin' % r.name)\n                vd.addGlobals({funcname: func})\n                setattr(vd, funcname, func)\n\n        # check for plugins with newer versions\n        def is_stale(r):\n            v = _loadedVersion(r)\n            return v and r.latest_ver and v != r.latest_ver\n\n        stale_plugins = list(filter(is_stale, self.rows))\n        if len(stale_plugins) > 0:\n            vd.warning(f'update available for {len(stale_plugins)} plugins')\n\n    def installPlugin(self, plugin):\n        # pip3 install requirements\n        initpath = _plugin_init()\n        os.makedirs(initpath.parent, exist_ok=True)\n        if not initpath.exists():\n            initpath.touch()\n\n        outpath = _plugin_path(plugin)\n        overwrite = True\n        if outpath.exists():\n            try:\n                vd.confirm(\"plugin path already exists, overwrite? \")\n            except ExpectedException:\n                overwrite = False\n                if _plugin_in_import_list(plugin):\n                    vd.fail(\"plugin already loaded\")\n                else:\n                    self._loadPlugin(plugin)\n        if overwrite:\n            self._install(plugin)\n\n    @asyncthread\n    def _install(self, plugin):\n        outpath = _plugin_path(plugin)\n\n        if \"git+\" in plugin.url:\n            p = subprocess.Popen([sys.executable, '-m', 'pip', 'install',\n                                '--upgrade', plugin.url],\n                    stdout=subprocess.PIPE,\n                    stderr=subprocess.PIPE)\n            out, err = p.communicate()\n            vd.status(out.decode())\n            if err:\n                vd.warning(err.decode())\n            if p.returncode != 0:\n                vd.fail('pip install failed')\n        else:\n            with vd.urlcache(plugin.url, days=0).open(encoding='utf-8') as pyfp:\n                contents = pyfp.read()\n                if plugin.sha256:\n                    if not _checkHash(contents, plugin.sha256):\n                        vd.error('%s plugin SHA256 does not match!' % plugin.name)\n                else:\n                    vd.warning('no SHA256 provided for %s plugin, not validating' % plugin.name)\n                with outpath.open(mode='w', encoding='utf-8') as outfp:\n                    outfp.write(contents)\n\n        if plugin.pydeps:\n            vd.pipinstall(plugin.pydeps.split())\n\n        vd.status('%s plugin installed' % plugin.name)\n\n        if _plugin_in_import_list(plugin):\n            vd.warning(\"plugin already loaded\")\n        else:\n            self._loadPlugin(plugin)\n\n\n    def _loadPlugin(self, plugin):\n        with Path(_plugin_init()).open(mode='a', encoding='utf-8') as fprc:\n            print(_plugin_import(plugin), file=fprc)\n            importlib.import_module(_plugin_import_name(plugin))\n            vd.status('%s plugin loaded' % plugin.name)\n\n\n    def removePluginIfExists(self, plugin):\n        self.removePlugin(plugin)\n\n    def removePlugin(self, plugin):\n        if not _plugin_in_import_list(plugin):\n            vd.fail(\"plugin not in import list\")\n\n        initpath = Path(_plugin_init())\n        oldinitpath = Path(initpath.with_suffix(initpath.suffix + '.bak'))\n        try:\n            shutil.copyfile(initpath, oldinitpath)\n\n            # Copy lines from the backup init file into its replacement, skipping lines that import the removed plugin.\n            #\n            # By matching from the start of a line through a word boundary, we avoid removing commented lines or inadvertently removing\n            # plugins with similar names.\n            with oldinitpath.open(encoding='utf-8') as old, initpath.open(mode='w', encoding='utf-8') as new:\n                r = re.compile(r'^{}\\W'.format(_plugin_import(plugin)))\n                new.writelines(line for line in old.readlines() if not r.match(line))\n\n            if os.path.exists(_plugin_path(plugin)):\n                os.unlink(_plugin_path(plugin))\n            sys.modules.pop(_plugin_import_name(plugin))\n            importlib.invalidate_caches()\n            vd.warning('{0} plugin uninstalled'.format(plugin['name']))\n        except FileNotFoundError:\n            vd.warning(\"no plugins/__init__.py found\")\n\n\nBaseSheet.addCommand(None, 'open-plugins', 'vd.push(vd.pluginsSheet)', 'Open Plugins Sheet to manage supported plugins')\n\nPluginsSheet.addCommand('a', 'add-plugin', 'installPlugin(cursorRow)', 'Install and enable current plugin')\nPluginsSheet.addCommand('d', 'delete-plugin', 'removePluginIfExists(cursorRow)', 'Disable current plugin')\n\nvd.addMenuItems('''\n    View > Plugins > open-plugins\n''')\n", "path": "visidata/plugins.py"}]}