We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
nikola check --clean-files should delete empty directories


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nikola/plugins/command/check.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 
3 # Copyright © 2012-2015 Roberto Alsina and others.
4 
5 # Permission is hereby granted, free of charge, to any
6 # person obtaining a copy of this software and associated
7 # documentation files (the "Software"), to deal in the
8 # Software without restriction, including without limitation
9 # the rights to use, copy, modify, merge, publish,
10 # distribute, sublicense, and/or sell copies of the
11 # Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
13 #
14 # The above copyright notice and this permission notice
15 # shall be included in all copies or substantial portions of
16 # the Software.
17 #
18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
19 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
20 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
21 # PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
22 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
23 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 
27 from __future__ import print_function
28 from collections import defaultdict
29 import os
30 import re
31 import sys
32 try:
33     from urllib import unquote
34     from urlparse import urlparse, urljoin, urldefrag
35 except ImportError:
36     from urllib.parse import unquote, urlparse, urljoin, urldefrag  # NOQA
37 
38 from doit.loader import generate_tasks
39 import lxml.html
40 import requests
41 
42 from nikola.plugin_categories import Command
43 from nikola.utils import get_logger, STDERR_HANDLER
44 
45 
46 def _call_nikola_list(site):
47     files = []
48     deps = defaultdict(list)
49     for task in generate_tasks('render_site', site.gen_tasks('render_site', "Task", '')):
50         files.extend(task.targets)
51         for target in task.targets:
52             deps[target].extend(task.file_dep)
53     for task in generate_tasks('post_render', site.gen_tasks('render_site', "LateTask", '')):
54         files.extend(task.targets)
55         for target in task.targets:
56             deps[target].extend(task.file_dep)
57     return files, deps
58 
59 
60 def real_scan_files(site):
61     task_fnames = set([])
62     real_fnames = set([])
63     output_folder = site.config['OUTPUT_FOLDER']
64     # First check that all targets are generated in the right places
65     for fname in _call_nikola_list(site)[0]:
66         fname = fname.strip()
67         if fname.startswith(output_folder):
68             task_fnames.add(fname)
69     # And now check that there are no non-target files
70     for root, dirs, files in os.walk(output_folder, followlinks=True):
71         for src_name in files:
72             fname = os.path.join(root, src_name)
73             real_fnames.add(fname)
74 
75     only_on_output = list(real_fnames - task_fnames)
76 
77     only_on_input = list(task_fnames - real_fnames)
78 
79     return (only_on_output, only_on_input)
80 
81 
82 def fs_relpath_from_url_path(url_path):
83     """Expects as input an urlparse(s).path"""
84     url_path = unquote(url_path)
85     # in windows relative paths don't begin with os.sep
86     if sys.platform == 'win32' and len(url_path):
87         url_path = url_path.replace('/', '\\')
88     return url_path
89 
90 
91 class CommandCheck(Command):
92     """Check the generated site."""
93 
94     name = "check"
95     logger = None
96 
97     doc_usage = "[-v] (-l [--find-sources] [-r] | -f [--clean-files])"
98     doc_purpose = "check links and files in the generated site"
99     cmd_options = [
100         {
101             'name': 'links',
102             'short': 'l',
103             'long': 'check-links',
104             'type': bool,
105             'default': False,
106             'help': 'Check for dangling links',
107         },
108         {
109             'name': 'files',
110             'short': 'f',
111             'long': 'check-files',
112             'type': bool,
113             'default': False,
114             'help': 'Check for unknown (orphaned and not generated) files',
115         },
116         {
117             'name': 'clean',
118             'long': 'clean-files',
119             'type': bool,
120             'default': False,
121             'help': 'Remove all unknown files, use with caution',
122         },
123         {
124             'name': 'find_sources',
125             'long': 'find-sources',
126             'type': bool,
127             'default': False,
128             'help': 'List possible source files for files with broken links.',
129         },
130         {
131             'name': 'verbose',
132             'long': 'verbose',
133             'short': 'v',
134             'type': bool,
135             'default': False,
136             'help': 'Be more verbose.',
137         },
138         {
139             'name': 'remote',
140             'long': 'remote',
141             'short': 'r',
142             'type': bool,
143             'default': False,
144             'help': 'Check that remote links work.',
145         },
146     ]
147 
148     def _execute(self, options, args):
149         """Check the generated site."""
150         self.logger = get_logger('check', STDERR_HANDLER)
151 
152         if not options['links'] and not options['files'] and not options['clean']:
153             print(self.help())
154             return False
155         if options['verbose']:
156             self.logger.level = 1
157         else:
158             self.logger.level = 4
159         if options['links']:
160             failure = self.scan_links(options['find_sources'], options['remote'])
161         if options['files']:
162             failure = self.scan_files()
163         if options['clean']:
164             failure = self.clean_files()
165         if failure:
166             return 1
167 
168     existing_targets = set([])
169     checked_remote_targets = {}
170 
171     def analyze(self, fname, find_sources=False, check_remote=False):
172         rv = False
173         self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']]
174         base_url = urlparse(self.site.config['BASE_URL'])
175         self.existing_targets.add(self.site.config['SITE_URL'])
176         self.existing_targets.add(self.site.config['BASE_URL'])
177         url_type = self.site.config['URL_TYPE']
178 
179         deps = {}
180         if find_sources:
181             deps = _call_nikola_list(self.site)[1]
182 
183         if url_type in ('absolute', 'full_path'):
184             url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path
185         try:
186             filename = fname
187 
188             if filename.startswith(self.site.config['CACHE_FOLDER']):
189                 # Do not look at links in the cache, which are not parsed by
190                 # anyone and may result in false positives.  Problems arise
191                 # with galleries, for example.  Full rationale: (Issue #1447)
192                 self.logger.notice("Ignoring {0} (in cache, links may be incorrect)".format(filename))
193                 return False
194 
195             if not os.path.exists(fname):
196                 # Quietly ignore files that don’t exist; use `nikola check -f` instead (Issue #1831)
197                 return False
198 
199             d = lxml.html.fromstring(open(filename, 'rb').read())
200             for l in d.iterlinks():
201                 target = l[2]
202                 if target == "#":
203                     continue
204                 target, _ = urldefrag(target)
205                 parsed = urlparse(target)
206 
207                 # Warn about links from https to http (mixed-security)
208                 if base_url.netloc == parsed.netloc and base_url.scheme == "https" and parsed.scheme == "http":
209                     self.logger.warn("Mixed-content security for link in {0}: {1}".format(filename, target))
210 
211                 # Absolute links to other domains, skip
212                 # Absolute links when using only paths, skip.
213                 if ((parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc) or \
214                         ((parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path')):
215                     if not check_remote or parsed.scheme not in ["http", "https"]:
216                         continue
217                     if parsed.netloc == base_url.netloc:  # absolute URL to self.site
218                         continue
219                     if target in self.checked_remote_targets:  # already checked this exact target
220                         if self.checked_remote_targets[target] > 399:
221                             self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target]))
222                         continue
223                     # Check the remote link works
224                     req_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0 (Nikola)'}  # I’m a real boy!
225                     resp = requests.head(target, headers=req_headers)
226                     self.checked_remote_targets[target] = resp.status_code
227                     if resp.status_code > 399:  # Error
228                         self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, resp.status_code))
229                         continue
230                     elif resp.status_code <= 399:  # The address leads *somewhere* that is not an error
231                         self.logger.debug("Successfully checked remote link in {0}: {1} [HTTP: {2}]".format(filename, target, resp.status_code))
232                         continue
233                     self.logger.warn("Could not check remote link in {0}: {1} [Unknown problem]".format(filename, target))
234                     continue
235 
236                 if url_type == 'rel_path':
237                     if target.startswith('/'):
238                         target_filename = os.path.abspath(
239                             os.path.join(self.site.config['OUTPUT_FOLDER'], unquote(target.lstrip('/'))))
240                     else:  # Relative path
241                         target_filename = os.path.abspath(
242                             os.path.join(os.path.dirname(filename), unquote(target)))
243 
244                 elif url_type in ('full_path', 'absolute'):
245                     if url_type == 'absolute':
246                         # convert to 'full_path' case, ie url relative to root
247                         url_rel_path = parsed.path[len(url_netloc_to_root):]
248                     else:
249                         # convert to relative to base path
250                         url_rel_path = target[len(url_netloc_to_root):]
251                     if url_rel_path == '' or url_rel_path.endswith('/'):
252                         url_rel_path = urljoin(url_rel_path, self.site.config['INDEX_FILE'])
253                     fs_rel_path = fs_relpath_from_url_path(url_rel_path)
254                     target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], fs_rel_path)
255 
256                 if any(re.search(x, target_filename) for x in self.whitelist):
257                     continue
258                 elif target_filename not in self.existing_targets:
259                     if os.path.exists(target_filename):
260                         self.logger.notice("Good link {0} => {1}".format(target, target_filename))
261                         self.existing_targets.add(target_filename)
262                     else:
263                         rv = True
264                         self.logger.warn("Broken link in {0}: {1}".format(filename, target))
265                         if find_sources:
266                             self.logger.warn("Possible sources:")
267                             self.logger.warn("\n".join(deps[filename]))
268                             self.logger.warn("===============================\n")
269         except Exception as exc:
270             self.logger.error("Error with: {0} {1}".format(filename, exc))
271         return rv
272 
273     def scan_links(self, find_sources=False, check_remote=False):
274         self.logger.info("Checking Links:")
275         self.logger.info("===============\n")
276         self.logger.notice("{0} mode".format(self.site.config['URL_TYPE']))
277         failure = False
278         # Maybe we should just examine all HTML files
279         output_folder = self.site.config['OUTPUT_FOLDER']
280         for fname in _call_nikola_list(self.site)[0]:
281             if fname.startswith(output_folder) and '.html' == fname[-5:]:
282                 if self.analyze(fname, find_sources, check_remote):
283                     failure = True
284         if not failure:
285             self.logger.info("All links checked.")
286         return failure
287 
288     def scan_files(self):
289         failure = False
290         self.logger.info("Checking Files:")
291         self.logger.info("===============\n")
292         only_on_output, only_on_input = real_scan_files(self.site)
293 
294         # Ignore folders
295         only_on_output = [p for p in only_on_output if not os.path.isdir(p)]
296         only_on_input = [p for p in only_on_input if not os.path.isdir(p)]
297 
298         if only_on_output:
299             only_on_output.sort()
300             self.logger.warn("Files from unknown origins (orphans):")
301             for f in only_on_output:
302                 self.logger.warn(f)
303             failure = True
304         if only_on_input:
305             only_on_input.sort()
306             self.logger.warn("Files not generated:")
307             for f in only_on_input:
308                 self.logger.warn(f)
309         if not failure:
310             self.logger.info("All files checked.")
311         return failure
312 
313     def clean_files(self):
314         only_on_output, _ = real_scan_files(self.site)
315         for f in only_on_output:
316             os.unlink(f)
317         return True
318
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py
--- a/nikola/plugins/command/check.py
+++ b/nikola/plugins/command/check.py
@@ -313,5 +313,19 @@
     def clean_files(self):
         only_on_output, _ = real_scan_files(self.site)
         for f in only_on_output:
+            self.logger.info('removed: {0}'.format(f))
             os.unlink(f)
+
+        # Find empty directories and remove them
+        output_folder = self.site.config['OUTPUT_FOLDER']
+        all_dirs = []
+        for root, dirs, files in os.walk(output_folder, followlinks=True):
+            all_dirs.append(root)
+        all_dirs.sort(key=len, reverse=True)
+        for d in all_dirs:
+            try:
+                os.rmdir(d)
+                self.logger.info('removed: {0}/'.format(d))
+            except OSError:
+                pass
         return True

{"golden_diff": "diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py\n--- a/nikola/plugins/command/check.py\n+++ b/nikola/plugins/command/check.py\n@@ -313,5 +313,19 @@\n     def clean_files(self):\n         only_on_output, _ = real_scan_files(self.site)\n         for f in only_on_output:\n+            self.logger.info('removed: {0}'.format(f))\n             os.unlink(f)\n+\n+        # Find empty directories and remove them\n+        output_folder = self.site.config['OUTPUT_FOLDER']\n+        all_dirs = []\n+        for root, dirs, files in os.walk(output_folder, followlinks=True):\n+            all_dirs.append(root)\n+        all_dirs.sort(key=len, reverse=True)\n+        for d in all_dirs:\n+            try:\n+                os.rmdir(d)\n+                self.logger.info('removed: {0}/'.format(d))\n+            except OSError:\n+                pass\n         return True\n", "issue": "nikola check --clean-files should delete empty directories\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n\n# Copyright \u00a9 2012-2015 Roberto Alsina and others.\n\n# Permission is hereby granted, free of charge, to any\n# person obtaining a copy of this software and associated\n# documentation files (the \"Software\"), to deal in the\n# Software without restriction, including without limitation\n# the rights to use, copy, modify, merge, publish,\n# distribute, sublicense, and/or sell copies of the\n# Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice\n# shall be included in all copies or substantial portions of\n# the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY\n# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE\n# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR\n# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS\n# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR\n# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\n# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\nfrom __future__ import print_function\nfrom collections import defaultdict\nimport os\nimport re\nimport sys\ntry:\n    from urllib import unquote\n    from urlparse import urlparse, urljoin, urldefrag\nexcept ImportError:\n    from urllib.parse import unquote, urlparse, urljoin, urldefrag  # NOQA\n\nfrom doit.loader import generate_tasks\nimport lxml.html\nimport requests\n\nfrom nikola.plugin_categories import Command\nfrom nikola.utils import get_logger, STDERR_HANDLER\n\n\ndef _call_nikola_list(site):\n    files = []\n    deps = defaultdict(list)\n    for task in generate_tasks('render_site', site.gen_tasks('render_site', \"Task\", '')):\n        files.extend(task.targets)\n        for target in task.targets:\n            deps[target].extend(task.file_dep)\n    for task in generate_tasks('post_render', site.gen_tasks('render_site', \"LateTask\", '')):\n        files.extend(task.targets)\n        for target in task.targets:\n            deps[target].extend(task.file_dep)\n    return files, deps\n\n\ndef real_scan_files(site):\n    task_fnames = set([])\n    real_fnames = set([])\n    output_folder = site.config['OUTPUT_FOLDER']\n    # First check that all targets are generated in the right places\n    for fname in _call_nikola_list(site)[0]:\n        fname = fname.strip()\n        if fname.startswith(output_folder):\n            task_fnames.add(fname)\n    # And now check that there are no non-target files\n    for root, dirs, files in os.walk(output_folder, followlinks=True):\n        for src_name in files:\n            fname = os.path.join(root, src_name)\n            real_fnames.add(fname)\n\n    only_on_output = list(real_fnames - task_fnames)\n\n    only_on_input = list(task_fnames - real_fnames)\n\n    return (only_on_output, only_on_input)\n\n\ndef fs_relpath_from_url_path(url_path):\n    \"\"\"Expects as input an urlparse(s).path\"\"\"\n    url_path = unquote(url_path)\n    # in windows relative paths don't begin with os.sep\n    if sys.platform == 'win32' and len(url_path):\n        url_path = url_path.replace('/', '\\\\')\n    return url_path\n\n\nclass CommandCheck(Command):\n    \"\"\"Check the generated site.\"\"\"\n\n    name = \"check\"\n    logger = None\n\n    doc_usage = \"[-v] (-l [--find-sources] [-r] | -f [--clean-files])\"\n    doc_purpose = \"check links and files in the generated site\"\n    cmd_options = [\n        {\n            'name': 'links',\n            'short': 'l',\n            'long': 'check-links',\n            'type': bool,\n            'default': False,\n            'help': 'Check for dangling links',\n        },\n        {\n            'name': 'files',\n            'short': 'f',\n            'long': 'check-files',\n            'type': bool,\n            'default': False,\n            'help': 'Check for unknown (orphaned and not generated) files',\n        },\n        {\n            'name': 'clean',\n            'long': 'clean-files',\n            'type': bool,\n            'default': False,\n            'help': 'Remove all unknown files, use with caution',\n        },\n        {\n            'name': 'find_sources',\n            'long': 'find-sources',\n            'type': bool,\n            'default': False,\n            'help': 'List possible source files for files with broken links.',\n        },\n        {\n            'name': 'verbose',\n            'long': 'verbose',\n            'short': 'v',\n            'type': bool,\n            'default': False,\n            'help': 'Be more verbose.',\n        },\n        {\n            'name': 'remote',\n            'long': 'remote',\n            'short': 'r',\n            'type': bool,\n            'default': False,\n            'help': 'Check that remote links work.',\n        },\n    ]\n\n    def _execute(self, options, args):\n        \"\"\"Check the generated site.\"\"\"\n        self.logger = get_logger('check', STDERR_HANDLER)\n\n        if not options['links'] and not options['files'] and not options['clean']:\n            print(self.help())\n            return False\n        if options['verbose']:\n            self.logger.level = 1\n        else:\n            self.logger.level = 4\n        if options['links']:\n            failure = self.scan_links(options['find_sources'], options['remote'])\n        if options['files']:\n            failure = self.scan_files()\n        if options['clean']:\n            failure = self.clean_files()\n        if failure:\n            return 1\n\n    existing_targets = set([])\n    checked_remote_targets = {}\n\n    def analyze(self, fname, find_sources=False, check_remote=False):\n        rv = False\n        self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']]\n        base_url = urlparse(self.site.config['BASE_URL'])\n        self.existing_targets.add(self.site.config['SITE_URL'])\n        self.existing_targets.add(self.site.config['BASE_URL'])\n        url_type = self.site.config['URL_TYPE']\n\n        deps = {}\n        if find_sources:\n            deps = _call_nikola_list(self.site)[1]\n\n        if url_type in ('absolute', 'full_path'):\n            url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path\n        try:\n            filename = fname\n\n            if filename.startswith(self.site.config['CACHE_FOLDER']):\n                # Do not look at links in the cache, which are not parsed by\n                # anyone and may result in false positives.  Problems arise\n                # with galleries, for example.  Full rationale: (Issue #1447)\n                self.logger.notice(\"Ignoring {0} (in cache, links may be incorrect)\".format(filename))\n                return False\n\n            if not os.path.exists(fname):\n                # Quietly ignore files that don\u2019t exist; use `nikola check -f` instead (Issue #1831)\n                return False\n\n            d = lxml.html.fromstring(open(filename, 'rb').read())\n            for l in d.iterlinks():\n                target = l[2]\n                if target == \"#\":\n                    continue\n                target, _ = urldefrag(target)\n                parsed = urlparse(target)\n\n                # Warn about links from https to http (mixed-security)\n                if base_url.netloc == parsed.netloc and base_url.scheme == \"https\" and parsed.scheme == \"http\":\n                    self.logger.warn(\"Mixed-content security for link in {0}: {1}\".format(filename, target))\n\n                # Absolute links to other domains, skip\n                # Absolute links when using only paths, skip.\n                if ((parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc) or \\\n                        ((parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path')):\n                    if not check_remote or parsed.scheme not in [\"http\", \"https\"]:\n                        continue\n                    if parsed.netloc == base_url.netloc:  # absolute URL to self.site\n                        continue\n                    if target in self.checked_remote_targets:  # already checked this exact target\n                        if self.checked_remote_targets[target] > 399:\n                            self.logger.warn(\"Broken link in {0}: {1} [Error {2}]\".format(filename, target, self.checked_remote_targets[target]))\n                        continue\n                    # Check the remote link works\n                    req_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0 (Nikola)'}  # I\u2019m a real boy!\n                    resp = requests.head(target, headers=req_headers)\n                    self.checked_remote_targets[target] = resp.status_code\n                    if resp.status_code > 399:  # Error\n                        self.logger.warn(\"Broken link in {0}: {1} [Error {2}]\".format(filename, target, resp.status_code))\n                        continue\n                    elif resp.status_code <= 399:  # The address leads *somewhere* that is not an error\n                        self.logger.debug(\"Successfully checked remote link in {0}: {1} [HTTP: {2}]\".format(filename, target, resp.status_code))\n                        continue\n                    self.logger.warn(\"Could not check remote link in {0}: {1} [Unknown problem]\".format(filename, target))\n                    continue\n\n                if url_type == 'rel_path':\n                    if target.startswith('/'):\n                        target_filename = os.path.abspath(\n                            os.path.join(self.site.config['OUTPUT_FOLDER'], unquote(target.lstrip('/'))))\n                    else:  # Relative path\n                        target_filename = os.path.abspath(\n                            os.path.join(os.path.dirname(filename), unquote(target)))\n\n                elif url_type in ('full_path', 'absolute'):\n                    if url_type == 'absolute':\n                        # convert to 'full_path' case, ie url relative to root\n                        url_rel_path = parsed.path[len(url_netloc_to_root):]\n                    else:\n                        # convert to relative to base path\n                        url_rel_path = target[len(url_netloc_to_root):]\n                    if url_rel_path == '' or url_rel_path.endswith('/'):\n                        url_rel_path = urljoin(url_rel_path, self.site.config['INDEX_FILE'])\n                    fs_rel_path = fs_relpath_from_url_path(url_rel_path)\n                    target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], fs_rel_path)\n\n                if any(re.search(x, target_filename) for x in self.whitelist):\n                    continue\n                elif target_filename not in self.existing_targets:\n                    if os.path.exists(target_filename):\n                        self.logger.notice(\"Good link {0} => {1}\".format(target, target_filename))\n                        self.existing_targets.add(target_filename)\n                    else:\n                        rv = True\n                        self.logger.warn(\"Broken link in {0}: {1}\".format(filename, target))\n                        if find_sources:\n                            self.logger.warn(\"Possible sources:\")\n                            self.logger.warn(\"\\n\".join(deps[filename]))\n                            self.logger.warn(\"===============================\\n\")\n        except Exception as exc:\n            self.logger.error(\"Error with: {0} {1}\".format(filename, exc))\n        return rv\n\n    def scan_links(self, find_sources=False, check_remote=False):\n        self.logger.info(\"Checking Links:\")\n        self.logger.info(\"===============\\n\")\n        self.logger.notice(\"{0} mode\".format(self.site.config['URL_TYPE']))\n        failure = False\n        # Maybe we should just examine all HTML files\n        output_folder = self.site.config['OUTPUT_FOLDER']\n        for fname in _call_nikola_list(self.site)[0]:\n            if fname.startswith(output_folder) and '.html' == fname[-5:]:\n                if self.analyze(fname, find_sources, check_remote):\n                    failure = True\n        if not failure:\n            self.logger.info(\"All links checked.\")\n        return failure\n\n    def scan_files(self):\n        failure = False\n        self.logger.info(\"Checking Files:\")\n        self.logger.info(\"===============\\n\")\n        only_on_output, only_on_input = real_scan_files(self.site)\n\n        # Ignore folders\n        only_on_output = [p for p in only_on_output if not os.path.isdir(p)]\n        only_on_input = [p for p in only_on_input if not os.path.isdir(p)]\n\n        if only_on_output:\n            only_on_output.sort()\n            self.logger.warn(\"Files from unknown origins (orphans):\")\n            for f in only_on_output:\n                self.logger.warn(f)\n            failure = True\n        if only_on_input:\n            only_on_input.sort()\n            self.logger.warn(\"Files not generated:\")\n            for f in only_on_input:\n                self.logger.warn(f)\n        if not failure:\n            self.logger.info(\"All files checked.\")\n        return failure\n\n    def clean_files(self):\n        only_on_output, _ = real_scan_files(self.site)\n        for f in only_on_output:\n            os.unlink(f)\n        return True\n", "path": "nikola/plugins/command/check.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n\n# Copyright \u00a9 2012-2015 Roberto Alsina and others.\n\n# Permission is hereby granted, free of charge, to any\n# person obtaining a copy of this software and associated\n# documentation files (the \"Software\"), to deal in the\n# Software without restriction, including without limitation\n# the rights to use, copy, modify, merge, publish,\n# distribute, sublicense, and/or sell copies of the\n# Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice\n# shall be included in all copies or substantial portions of\n# the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY\n# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE\n# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR\n# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS\n# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR\n# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\n# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\nfrom __future__ import print_function\nfrom collections import defaultdict\nimport os\nimport re\nimport sys\ntry:\n    from urllib import unquote\n    from urlparse import urlparse, urljoin, urldefrag\nexcept ImportError:\n    from urllib.parse import unquote, urlparse, urljoin, urldefrag  # NOQA\n\nfrom doit.loader import generate_tasks\nimport lxml.html\nimport requests\n\nfrom nikola.plugin_categories import Command\nfrom nikola.utils import get_logger, STDERR_HANDLER\n\n\ndef _call_nikola_list(site):\n    files = []\n    deps = defaultdict(list)\n    for task in generate_tasks('render_site', site.gen_tasks('render_site', \"Task\", '')):\n        files.extend(task.targets)\n        for target in task.targets:\n            deps[target].extend(task.file_dep)\n    for task in generate_tasks('post_render', site.gen_tasks('render_site', \"LateTask\", '')):\n        files.extend(task.targets)\n        for target in task.targets:\n            deps[target].extend(task.file_dep)\n    return files, deps\n\n\ndef real_scan_files(site):\n    task_fnames = set([])\n    real_fnames = set([])\n    output_folder = site.config['OUTPUT_FOLDER']\n    # First check that all targets are generated in the right places\n    for fname in _call_nikola_list(site)[0]:\n        fname = fname.strip()\n        if fname.startswith(output_folder):\n            task_fnames.add(fname)\n    # And now check that there are no non-target files\n    for root, dirs, files in os.walk(output_folder, followlinks=True):\n        for src_name in files:\n            fname = os.path.join(root, src_name)\n            real_fnames.add(fname)\n\n    only_on_output = list(real_fnames - task_fnames)\n\n    only_on_input = list(task_fnames - real_fnames)\n\n    return (only_on_output, only_on_input)\n\n\ndef fs_relpath_from_url_path(url_path):\n    \"\"\"Expects as input an urlparse(s).path\"\"\"\n    url_path = unquote(url_path)\n    # in windows relative paths don't begin with os.sep\n    if sys.platform == 'win32' and len(url_path):\n        url_path = url_path.replace('/', '\\\\')\n    return url_path\n\n\nclass CommandCheck(Command):\n    \"\"\"Check the generated site.\"\"\"\n\n    name = \"check\"\n    logger = None\n\n    doc_usage = \"[-v] (-l [--find-sources] [-r] | -f [--clean-files])\"\n    doc_purpose = \"check links and files in the generated site\"\n    cmd_options = [\n        {\n            'name': 'links',\n            'short': 'l',\n            'long': 'check-links',\n            'type': bool,\n            'default': False,\n            'help': 'Check for dangling links',\n        },\n        {\n            'name': 'files',\n            'short': 'f',\n            'long': 'check-files',\n            'type': bool,\n            'default': False,\n            'help': 'Check for unknown (orphaned and not generated) files',\n        },\n        {\n            'name': 'clean',\n            'long': 'clean-files',\n            'type': bool,\n            'default': False,\n            'help': 'Remove all unknown files, use with caution',\n        },\n        {\n            'name': 'find_sources',\n            'long': 'find-sources',\n            'type': bool,\n            'default': False,\n            'help': 'List possible source files for files with broken links.',\n        },\n        {\n            'name': 'verbose',\n            'long': 'verbose',\n            'short': 'v',\n            'type': bool,\n            'default': False,\n            'help': 'Be more verbose.',\n        },\n        {\n            'name': 'remote',\n            'long': 'remote',\n            'short': 'r',\n            'type': bool,\n            'default': False,\n            'help': 'Check that remote links work.',\n        },\n    ]\n\n    def _execute(self, options, args):\n        \"\"\"Check the generated site.\"\"\"\n        self.logger = get_logger('check', STDERR_HANDLER)\n\n        if not options['links'] and not options['files'] and not options['clean']:\n            print(self.help())\n            return False\n        if options['verbose']:\n            self.logger.level = 1\n        else:\n            self.logger.level = 4\n        if options['links']:\n            failure = self.scan_links(options['find_sources'], options['remote'])\n        if options['files']:\n            failure = self.scan_files()\n        if options['clean']:\n            failure = self.clean_files()\n        if failure:\n            return 1\n\n    existing_targets = set([])\n    checked_remote_targets = {}\n\n    def analyze(self, fname, find_sources=False, check_remote=False):\n        rv = False\n        self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']]\n        base_url = urlparse(self.site.config['BASE_URL'])\n        self.existing_targets.add(self.site.config['SITE_URL'])\n        self.existing_targets.add(self.site.config['BASE_URL'])\n        url_type = self.site.config['URL_TYPE']\n\n        deps = {}\n        if find_sources:\n            deps = _call_nikola_list(self.site)[1]\n\n        if url_type in ('absolute', 'full_path'):\n            url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path\n        try:\n            filename = fname\n\n            if filename.startswith(self.site.config['CACHE_FOLDER']):\n                # Do not look at links in the cache, which are not parsed by\n                # anyone and may result in false positives.  Problems arise\n                # with galleries, for example.  Full rationale: (Issue #1447)\n                self.logger.notice(\"Ignoring {0} (in cache, links may be incorrect)\".format(filename))\n                return False\n\n            if not os.path.exists(fname):\n                # Quietly ignore files that don\u2019t exist; use `nikola check -f` instead (Issue #1831)\n                return False\n\n            d = lxml.html.fromstring(open(filename, 'rb').read())\n            for l in d.iterlinks():\n                target = l[2]\n                if target == \"#\":\n                    continue\n                target, _ = urldefrag(target)\n                parsed = urlparse(target)\n\n                # Warn about links from https to http (mixed-security)\n                if base_url.netloc == parsed.netloc and base_url.scheme == \"https\" and parsed.scheme == \"http\":\n                    self.logger.warn(\"Mixed-content security for link in {0}: {1}\".format(filename, target))\n\n                # Absolute links to other domains, skip\n                # Absolute links when using only paths, skip.\n                if ((parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc) or \\\n                        ((parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path')):\n                    if not check_remote or parsed.scheme not in [\"http\", \"https\"]:\n                        continue\n                    if parsed.netloc == base_url.netloc:  # absolute URL to self.site\n                        continue\n                    if target in self.checked_remote_targets:  # already checked this exact target\n                        if self.checked_remote_targets[target] > 399:\n                            self.logger.warn(\"Broken link in {0}: {1} [Error {2}]\".format(filename, target, self.checked_remote_targets[target]))\n                        continue\n                    # Check the remote link works\n                    req_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0 (Nikola)'}  # I\u2019m a real boy!\n                    resp = requests.head(target, headers=req_headers)\n                    self.checked_remote_targets[target] = resp.status_code\n                    if resp.status_code > 399:  # Error\n                        self.logger.warn(\"Broken link in {0}: {1} [Error {2}]\".format(filename, target, resp.status_code))\n                        continue\n                    elif resp.status_code <= 399:  # The address leads *somewhere* that is not an error\n                        self.logger.debug(\"Successfully checked remote link in {0}: {1} [HTTP: {2}]\".format(filename, target, resp.status_code))\n                        continue\n                    self.logger.warn(\"Could not check remote link in {0}: {1} [Unknown problem]\".format(filename, target))\n                    continue\n\n                if url_type == 'rel_path':\n                    if target.startswith('/'):\n                        target_filename = os.path.abspath(\n                            os.path.join(self.site.config['OUTPUT_FOLDER'], unquote(target.lstrip('/'))))\n                    else:  # Relative path\n                        target_filename = os.path.abspath(\n                            os.path.join(os.path.dirname(filename), unquote(target)))\n\n                elif url_type in ('full_path', 'absolute'):\n                    if url_type == 'absolute':\n                        # convert to 'full_path' case, ie url relative to root\n                        url_rel_path = parsed.path[len(url_netloc_to_root):]\n                    else:\n                        # convert to relative to base path\n                        url_rel_path = target[len(url_netloc_to_root):]\n                    if url_rel_path == '' or url_rel_path.endswith('/'):\n                        url_rel_path = urljoin(url_rel_path, self.site.config['INDEX_FILE'])\n                    fs_rel_path = fs_relpath_from_url_path(url_rel_path)\n                    target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], fs_rel_path)\n\n                if any(re.search(x, target_filename) for x in self.whitelist):\n                    continue\n                elif target_filename not in self.existing_targets:\n                    if os.path.exists(target_filename):\n                        self.logger.notice(\"Good link {0} => {1}\".format(target, target_filename))\n                        self.existing_targets.add(target_filename)\n                    else:\n                        rv = True\n                        self.logger.warn(\"Broken link in {0}: {1}\".format(filename, target))\n                        if find_sources:\n                            self.logger.warn(\"Possible sources:\")\n                            self.logger.warn(\"\\n\".join(deps[filename]))\n                            self.logger.warn(\"===============================\\n\")\n        except Exception as exc:\n            self.logger.error(\"Error with: {0} {1}\".format(filename, exc))\n        return rv\n\n    def scan_links(self, find_sources=False, check_remote=False):\n        self.logger.info(\"Checking Links:\")\n        self.logger.info(\"===============\\n\")\n        self.logger.notice(\"{0} mode\".format(self.site.config['URL_TYPE']))\n        failure = False\n        # Maybe we should just examine all HTML files\n        output_folder = self.site.config['OUTPUT_FOLDER']\n        for fname in _call_nikola_list(self.site)[0]:\n            if fname.startswith(output_folder) and '.html' == fname[-5:]:\n                if self.analyze(fname, find_sources, check_remote):\n                    failure = True\n        if not failure:\n            self.logger.info(\"All links checked.\")\n        return failure\n\n    def scan_files(self):\n        failure = False\n        self.logger.info(\"Checking Files:\")\n        self.logger.info(\"===============\\n\")\n        only_on_output, only_on_input = real_scan_files(self.site)\n\n        # Ignore folders\n        only_on_output = [p for p in only_on_output if not os.path.isdir(p)]\n        only_on_input = [p for p in only_on_input if not os.path.isdir(p)]\n\n        if only_on_output:\n            only_on_output.sort()\n            self.logger.warn(\"Files from unknown origins (orphans):\")\n            for f in only_on_output:\n                self.logger.warn(f)\n            failure = True\n        if only_on_input:\n            only_on_input.sort()\n            self.logger.warn(\"Files not generated:\")\n            for f in only_on_input:\n                self.logger.warn(f)\n        if not failure:\n            self.logger.info(\"All files checked.\")\n        return failure\n\n    def clean_files(self):\n        only_on_output, _ = real_scan_files(self.site)\n        for f in only_on_output:\n            self.logger.info('removed: {0}'.format(f))\n            os.unlink(f)\n\n        # Find empty directories and remove them\n        output_folder = self.site.config['OUTPUT_FOLDER']\n        all_dirs = []\n        for root, dirs, files in os.walk(output_folder, followlinks=True):\n            all_dirs.append(root)\n        all_dirs.sort(key=len, reverse=True)\n        for d in all_dirs:\n            try:\n                os.rmdir(d)\n                self.logger.info('removed: {0}/'.format(d))\n            except OSError:\n                pass\n        return True\n", "path": "nikola/plugins/command/check.py"}]}