We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Sitemaps are not rebuilt
`touch pages/test-1.html
nikola build
mv pages/test-1.html pages/test-2.html
nikola build`

sitemap.xml is not rebuilt. Contains test-1.html (expected) but not test-2.html (woopsy). Output contains both (expected).

`nikola check --clean-files`

Removes test-1.html from output but also does not rebuild sitemap.xml.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nikola/plugins/task/sitemap/__init__.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 
3 # Copyright © 2012-2014 Roberto Alsina and others.
4 
5 # Permission is hereby granted, free of charge, to any
6 # person obtaining a copy of this software and associated
7 # documentation files (the "Software"), to deal in the
8 # Software without restriction, including without limitation
9 # the rights to use, copy, modify, merge, publish,
10 # distribute, sublicense, and/or sell copies of the
11 # Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
13 #
14 # The above copyright notice and this permission notice
15 # shall be included in all copies or substantial portions of
16 # the Software.
17 #
18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
19 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
20 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
21 # PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
22 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
23 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 
27 from __future__ import print_function, absolute_import, unicode_literals
28 import io
29 import datetime
30 import os
31 try:
32     from urlparse import urljoin, urlparse
33     import robotparser as robotparser
34 except ImportError:
35     from urllib.parse import urljoin, urlparse  # NOQA
36     import urllib.robotparser as robotparser  # NOQA
37 
38 from nikola.plugin_categories import LateTask
39 from nikola.utils import config_changed
40 
41 
42 urlset_header = """<?xml version="1.0" encoding="UTF-8"?>
43 <urlset
44     xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
45     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
46     xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9
47                         http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
48 """
49 
50 loc_format = """ <url>
51   <loc>{0}</loc>
52   <lastmod>{1}</lastmod>
53  </url>
54 """
55 
56 urlset_footer = "</urlset>"
57 
58 sitemapindex_header = """<?xml version="1.0" encoding="UTF-8"?>
59 <sitemapindex
60     xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
61     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
62     xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9
63                         http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
64 """
65 
66 sitemap_format = """ <sitemap>
67   <loc>{0}</loc>
68   <lastmod>{1}</lastmod>
69  </sitemap>
70 """
71 
72 sitemapindex_footer = "</sitemapindex>"
73 
74 
75 def get_base_path(base):
76     """returns the path of a base URL if it contains one.
77 
78     >>> get_base_path('http://some.site') == '/'
79     True
80     >>> get_base_path('http://some.site/') == '/'
81     True
82     >>> get_base_path('http://some.site/some/sub-path') == '/some/sub-path/'
83     True
84     >>> get_base_path('http://some.site/some/sub-path/') == '/some/sub-path/'
85     True
86     """
87     # first parse the base_url for some path
88     base_parsed = urlparse(base)
89 
90     if not base_parsed.path:
91         sub_path = ''
92     else:
93         sub_path = base_parsed.path
94     if sub_path.endswith('/'):
95         return sub_path
96     else:
97         return sub_path + '/'
98 
99 
100 class Sitemap(LateTask):
101     """Generate a sitemap."""
102 
103     name = "sitemap"
104 
105     def gen_tasks(self):
106         """Generate a sitemap."""
107         kw = {
108             "base_url": self.site.config["BASE_URL"],
109             "site_url": self.site.config["SITE_URL"],
110             "output_folder": self.site.config["OUTPUT_FOLDER"],
111             "strip_indexes": self.site.config["STRIP_INDEXES"],
112             "index_file": self.site.config["INDEX_FILE"],
113             "sitemap_include_fileless_dirs": self.site.config["SITEMAP_INCLUDE_FILELESS_DIRS"],
114             "mapped_extensions": self.site.config.get('MAPPED_EXTENSIONS', ['.html', '.htm', '.xml', '.rss']),
115             "robots_exclusions": self.site.config["ROBOTS_EXCLUSIONS"]
116         }
117 
118         output = kw['output_folder']
119         base_url = kw['base_url']
120         mapped_exts = kw['mapped_extensions']
121 
122         output_path = kw['output_folder']
123         sitemapindex_path = os.path.join(output_path, "sitemapindex.xml")
124         sitemap_path = os.path.join(output_path, "sitemap.xml")
125         base_path = get_base_path(kw['base_url'])
126         sitemapindex = {}
127         urlset = {}
128 
129         def scan_locs():
130             for root, dirs, files in os.walk(output, followlinks=True):
131                 if not dirs and not files and not kw['sitemap_include_fileless_dirs']:
132                     continue  # Totally empty, not on sitemap
133                 path = os.path.relpath(root, output)
134                 # ignore the current directory.
135                 path = (path.replace(os.sep, '/') + '/').replace('./', '')
136                 lastmod = self.get_lastmod(root)
137                 loc = urljoin(base_url, base_path + path)
138                 if kw['index_file'] in files and kw['strip_indexes']:  # ignore folders when not stripping urls
139                     urlset[loc] = loc_format.format(loc, lastmod)
140                 for fname in files:
141                     if kw['strip_indexes'] and fname == kw['index_file']:
142                         continue  # We already mapped the folder
143                     if os.path.splitext(fname)[-1] in mapped_exts:
144                         real_path = os.path.join(root, fname)
145                         path = os.path.relpath(real_path, output)
146                         if path.endswith(kw['index_file']) and kw['strip_indexes']:
147                             # ignore index files when stripping urls
148                             continue
149                         if not robot_fetch(path):
150                             continue
151                         if path.endswith('.html') or path.endswith('.htm'):
152                             try:
153                                 if u'<!doctype html' not in io.open(real_path, 'r', encoding='utf8').read(1024).lower():
154                                     # ignores "html" files without doctype
155                                     # alexa-verify, google-site-verification, etc.
156                                     continue
157                             except UnicodeDecodeError:
158                                 # ignore ancient files
159                                 # most non-utf8 files are worthless anyways
160                                 continue
161                         """ put RSS in sitemapindex[] instead of in urlset[], sitemap_path is included after it is generated """
162                         if path.endswith('.xml') or path.endswith('.rss'):
163                             filehead = io.open(real_path, 'r', encoding='utf8').read(512)
164                             if u'<rss' in filehead or (u'<urlset' in filehead and path != sitemap_path):
165                                 path = path.replace(os.sep, '/')
166                                 lastmod = self.get_lastmod(real_path)
167                                 loc = urljoin(base_url, base_path + path)
168                                 sitemapindex[loc] = sitemap_format.format(loc, lastmod)
169                                 continue
170                             else:
171                                 continue  # ignores all XML files except those presumed to be RSS
172                         post = self.site.post_per_file.get(path)
173                         if post and (post.is_draft or post.is_private or post.publish_later):
174                             continue
175                         path = path.replace(os.sep, '/')
176                         lastmod = self.get_lastmod(real_path)
177                         loc = urljoin(base_url, base_path + path)
178                         urlset[loc] = loc_format.format(loc, lastmod)
179 
180         def robot_fetch(path):
181             for rule in kw["robots_exclusions"]:
182                 robot = robotparser.RobotFileParser()
183                 robot.parse(["User-Agent: *", "Disallow: {0}".format(rule)])
184                 if not robot.can_fetch("*", '/' + path):
185                     return False  # not robot food
186             return True
187 
188         def write_sitemap():
189             # Have to rescan, because files may have been added between
190             # task dep scanning and task execution
191             with io.open(sitemap_path, 'w+', encoding='utf8') as outf:
192                 outf.write(urlset_header)
193                 for k in sorted(urlset.keys()):
194                     outf.write(urlset[k])
195                 outf.write(urlset_footer)
196             sitemap_url = urljoin(base_url, base_path + "sitemap.xml")
197             sitemapindex[sitemap_url] = sitemap_format.format(sitemap_url, self.get_lastmod(sitemap_path))
198 
199         def write_sitemapindex():
200             with io.open(sitemapindex_path, 'w+', encoding='utf8') as outf:
201                 outf.write(sitemapindex_header)
202                 for k in sorted(sitemapindex.keys()):
203                     outf.write(sitemapindex[k])
204                 outf.write(sitemapindex_footer)
205 
206         # Yield a task to calculate the dependencies of the sitemap
207         # Other tasks can depend on this output, instead of having
208         # to scan locations.
209         def scan_locs_task():
210             scan_locs()
211             return {'locations': list(urlset.keys()) + list(sitemapindex.keys())}
212 
213         yield {
214             "basename": "_scan_locs",
215             "name": "sitemap",
216             "actions": [(scan_locs_task)]
217         }
218 
219         yield self.group_task()
220         yield {
221             "basename": "sitemap",
222             "name": sitemap_path,
223             "targets": [sitemap_path],
224             "actions": [(write_sitemap,)],
225             "uptodate": [config_changed(kw)],
226             "clean": True,
227             "task_dep": ["render_site"],
228             "calc_dep": ["_scan_locs:sitemap"],
229         }
230         yield {
231             "basename": "sitemap",
232             "name": sitemapindex_path,
233             "targets": [sitemapindex_path],
234             "actions": [(write_sitemapindex,)],
235             "uptodate": [config_changed(kw)],
236             "clean": True,
237             "file_dep": [sitemap_path]
238         }
239 
240     def get_lastmod(self, p):
241         if self.site.invariant:
242             return '2014-01-01'
243         else:
244             return datetime.datetime.fromtimestamp(os.stat(p).st_mtime).isoformat().split('T')[0]
245 
246 if __name__ == '__main__':
247     import doctest
248     doctest.testmod()
249
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nikola/plugins/task/sitemap/__init__.py b/nikola/plugins/task/sitemap/__init__.py
--- a/nikola/plugins/task/sitemap/__init__.py
+++ b/nikola/plugins/task/sitemap/__init__.py
@@ -208,7 +208,27 @@
         # to scan locations.
         def scan_locs_task():
             scan_locs()
-            return {'locations': list(urlset.keys()) + list(sitemapindex.keys())}
+
+            # Generate a list of file dependencies for the actual generation
+            # task, so rebuilds are triggered.  (Issue #1032)
+            output = kw["output_folder"]
+            file_dep = []
+
+            for i in urlset.keys():
+                p = os.path.join(output, urlparse(i).path.lstrip('/'))
+                if not p.endswith('sitemap.xml') and not p.endswith('/'):
+                    file_dep.append(p)
+                if p.endswith('/') and os.path.exists(p + 'index.html'):
+                    file_dep.append(p + 'index.html')
+
+            for i in sitemapindex.keys():
+                p = os.path.join(output, urlparse(i).path.lstrip('/'))
+                if not p.endswith('sitemap.xml') and not p.endswith('/'):
+                    file_dep.append(p)
+                if p.endswith('/') and os.path.exists(p + 'index.html'):
+                    file_dep.append(p + 'index.html')
+
+            return {'file_dep': file_dep}
 
         yield {
             "basename": "_scan_locs",

{"golden_diff": "diff --git a/nikola/plugins/task/sitemap/__init__.py b/nikola/plugins/task/sitemap/__init__.py\n--- a/nikola/plugins/task/sitemap/__init__.py\n+++ b/nikola/plugins/task/sitemap/__init__.py\n@@ -208,7 +208,27 @@\n         # to scan locations.\n         def scan_locs_task():\n             scan_locs()\n-            return {'locations': list(urlset.keys()) + list(sitemapindex.keys())}\n+\n+            # Generate a list of file dependencies for the actual generation\n+            # task, so rebuilds are triggered.  (Issue #1032)\n+            output = kw[\"output_folder\"]\n+            file_dep = []\n+\n+            for i in urlset.keys():\n+                p = os.path.join(output, urlparse(i).path.lstrip('/'))\n+                if not p.endswith('sitemap.xml') and not p.endswith('/'):\n+                    file_dep.append(p)\n+                if p.endswith('/') and os.path.exists(p + 'index.html'):\n+                    file_dep.append(p + 'index.html')\n+\n+            for i in sitemapindex.keys():\n+                p = os.path.join(output, urlparse(i).path.lstrip('/'))\n+                if not p.endswith('sitemap.xml') and not p.endswith('/'):\n+                    file_dep.append(p)\n+                if p.endswith('/') and os.path.exists(p + 'index.html'):\n+                    file_dep.append(p + 'index.html')\n+\n+            return {'file_dep': file_dep}\n \n         yield {\n             \"basename\": \"_scan_locs\",\n", "issue": "Sitemaps are not rebuilt\n`touch pages/test-1.html\nnikola build\nmv pages/test-1.html pages/test-2.html\nnikola build`\n\nsitemap.xml is not rebuilt. Contains test-1.html (expected) but not test-2.html (woopsy). Output contains both (expected).\n\n`nikola check --clean-files`\n\nRemoves test-1.html from output but also does not rebuild sitemap.xml.\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n\n# Copyright \u00a9 2012-2014 Roberto Alsina and others.\n\n# Permission is hereby granted, free of charge, to any\n# person obtaining a copy of this software and associated\n# documentation files (the \"Software\"), to deal in the\n# Software without restriction, including without limitation\n# the rights to use, copy, modify, merge, publish,\n# distribute, sublicense, and/or sell copies of the\n# Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice\n# shall be included in all copies or substantial portions of\n# the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY\n# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE\n# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR\n# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS\n# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR\n# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\n# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\nfrom __future__ import print_function, absolute_import, unicode_literals\nimport io\nimport datetime\nimport os\ntry:\n    from urlparse import urljoin, urlparse\n    import robotparser as robotparser\nexcept ImportError:\n    from urllib.parse import urljoin, urlparse  # NOQA\n    import urllib.robotparser as robotparser  # NOQA\n\nfrom nikola.plugin_categories import LateTask\nfrom nikola.utils import config_changed\n\n\nurlset_header = \"\"\"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<urlset\n    xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n    xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9\n                        http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">\n\"\"\"\n\nloc_format = \"\"\" <url>\n  <loc>{0}</loc>\n  <lastmod>{1}</lastmod>\n </url>\n\"\"\"\n\nurlset_footer = \"</urlset>\"\n\nsitemapindex_header = \"\"\"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<sitemapindex\n    xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n    xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9\n                        http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">\n\"\"\"\n\nsitemap_format = \"\"\" <sitemap>\n  <loc>{0}</loc>\n  <lastmod>{1}</lastmod>\n </sitemap>\n\"\"\"\n\nsitemapindex_footer = \"</sitemapindex>\"\n\n\ndef get_base_path(base):\n    \"\"\"returns the path of a base URL if it contains one.\n\n    >>> get_base_path('http://some.site') == '/'\n    True\n    >>> get_base_path('http://some.site/') == '/'\n    True\n    >>> get_base_path('http://some.site/some/sub-path') == '/some/sub-path/'\n    True\n    >>> get_base_path('http://some.site/some/sub-path/') == '/some/sub-path/'\n    True\n    \"\"\"\n    # first parse the base_url for some path\n    base_parsed = urlparse(base)\n\n    if not base_parsed.path:\n        sub_path = ''\n    else:\n        sub_path = base_parsed.path\n    if sub_path.endswith('/'):\n        return sub_path\n    else:\n        return sub_path + '/'\n\n\nclass Sitemap(LateTask):\n    \"\"\"Generate a sitemap.\"\"\"\n\n    name = \"sitemap\"\n\n    def gen_tasks(self):\n        \"\"\"Generate a sitemap.\"\"\"\n        kw = {\n            \"base_url\": self.site.config[\"BASE_URL\"],\n            \"site_url\": self.site.config[\"SITE_URL\"],\n            \"output_folder\": self.site.config[\"OUTPUT_FOLDER\"],\n            \"strip_indexes\": self.site.config[\"STRIP_INDEXES\"],\n            \"index_file\": self.site.config[\"INDEX_FILE\"],\n            \"sitemap_include_fileless_dirs\": self.site.config[\"SITEMAP_INCLUDE_FILELESS_DIRS\"],\n            \"mapped_extensions\": self.site.config.get('MAPPED_EXTENSIONS', ['.html', '.htm', '.xml', '.rss']),\n            \"robots_exclusions\": self.site.config[\"ROBOTS_EXCLUSIONS\"]\n        }\n\n        output = kw['output_folder']\n        base_url = kw['base_url']\n        mapped_exts = kw['mapped_extensions']\n\n        output_path = kw['output_folder']\n        sitemapindex_path = os.path.join(output_path, \"sitemapindex.xml\")\n        sitemap_path = os.path.join(output_path, \"sitemap.xml\")\n        base_path = get_base_path(kw['base_url'])\n        sitemapindex = {}\n        urlset = {}\n\n        def scan_locs():\n            for root, dirs, files in os.walk(output, followlinks=True):\n                if not dirs and not files and not kw['sitemap_include_fileless_dirs']:\n                    continue  # Totally empty, not on sitemap\n                path = os.path.relpath(root, output)\n                # ignore the current directory.\n                path = (path.replace(os.sep, '/') + '/').replace('./', '')\n                lastmod = self.get_lastmod(root)\n                loc = urljoin(base_url, base_path + path)\n                if kw['index_file'] in files and kw['strip_indexes']:  # ignore folders when not stripping urls\n                    urlset[loc] = loc_format.format(loc, lastmod)\n                for fname in files:\n                    if kw['strip_indexes'] and fname == kw['index_file']:\n                        continue  # We already mapped the folder\n                    if os.path.splitext(fname)[-1] in mapped_exts:\n                        real_path = os.path.join(root, fname)\n                        path = os.path.relpath(real_path, output)\n                        if path.endswith(kw['index_file']) and kw['strip_indexes']:\n                            # ignore index files when stripping urls\n                            continue\n                        if not robot_fetch(path):\n                            continue\n                        if path.endswith('.html') or path.endswith('.htm'):\n                            try:\n                                if u'<!doctype html' not in io.open(real_path, 'r', encoding='utf8').read(1024).lower():\n                                    # ignores \"html\" files without doctype\n                                    # alexa-verify, google-site-verification, etc.\n                                    continue\n                            except UnicodeDecodeError:\n                                # ignore ancient files\n                                # most non-utf8 files are worthless anyways\n                                continue\n                        \"\"\" put RSS in sitemapindex[] instead of in urlset[], sitemap_path is included after it is generated \"\"\"\n                        if path.endswith('.xml') or path.endswith('.rss'):\n                            filehead = io.open(real_path, 'r', encoding='utf8').read(512)\n                            if u'<rss' in filehead or (u'<urlset' in filehead and path != sitemap_path):\n                                path = path.replace(os.sep, '/')\n                                lastmod = self.get_lastmod(real_path)\n                                loc = urljoin(base_url, base_path + path)\n                                sitemapindex[loc] = sitemap_format.format(loc, lastmod)\n                                continue\n                            else:\n                                continue  # ignores all XML files except those presumed to be RSS\n                        post = self.site.post_per_file.get(path)\n                        if post and (post.is_draft or post.is_private or post.publish_later):\n                            continue\n                        path = path.replace(os.sep, '/')\n                        lastmod = self.get_lastmod(real_path)\n                        loc = urljoin(base_url, base_path + path)\n                        urlset[loc] = loc_format.format(loc, lastmod)\n\n        def robot_fetch(path):\n            for rule in kw[\"robots_exclusions\"]:\n                robot = robotparser.RobotFileParser()\n                robot.parse([\"User-Agent: *\", \"Disallow: {0}\".format(rule)])\n                if not robot.can_fetch(\"*\", '/' + path):\n                    return False  # not robot food\n            return True\n\n        def write_sitemap():\n            # Have to rescan, because files may have been added between\n            # task dep scanning and task execution\n            with io.open(sitemap_path, 'w+', encoding='utf8') as outf:\n                outf.write(urlset_header)\n                for k in sorted(urlset.keys()):\n                    outf.write(urlset[k])\n                outf.write(urlset_footer)\n            sitemap_url = urljoin(base_url, base_path + \"sitemap.xml\")\n            sitemapindex[sitemap_url] = sitemap_format.format(sitemap_url, self.get_lastmod(sitemap_path))\n\n        def write_sitemapindex():\n            with io.open(sitemapindex_path, 'w+', encoding='utf8') as outf:\n                outf.write(sitemapindex_header)\n                for k in sorted(sitemapindex.keys()):\n                    outf.write(sitemapindex[k])\n                outf.write(sitemapindex_footer)\n\n        # Yield a task to calculate the dependencies of the sitemap\n        # Other tasks can depend on this output, instead of having\n        # to scan locations.\n        def scan_locs_task():\n            scan_locs()\n            return {'locations': list(urlset.keys()) + list(sitemapindex.keys())}\n\n        yield {\n            \"basename\": \"_scan_locs\",\n            \"name\": \"sitemap\",\n            \"actions\": [(scan_locs_task)]\n        }\n\n        yield self.group_task()\n        yield {\n            \"basename\": \"sitemap\",\n            \"name\": sitemap_path,\n            \"targets\": [sitemap_path],\n            \"actions\": [(write_sitemap,)],\n            \"uptodate\": [config_changed(kw)],\n            \"clean\": True,\n            \"task_dep\": [\"render_site\"],\n            \"calc_dep\": [\"_scan_locs:sitemap\"],\n        }\n        yield {\n            \"basename\": \"sitemap\",\n            \"name\": sitemapindex_path,\n            \"targets\": [sitemapindex_path],\n            \"actions\": [(write_sitemapindex,)],\n            \"uptodate\": [config_changed(kw)],\n            \"clean\": True,\n            \"file_dep\": [sitemap_path]\n        }\n\n    def get_lastmod(self, p):\n        if self.site.invariant:\n            return '2014-01-01'\n        else:\n            return datetime.datetime.fromtimestamp(os.stat(p).st_mtime).isoformat().split('T')[0]\n\nif __name__ == '__main__':\n    import doctest\n    doctest.testmod()\n", "path": "nikola/plugins/task/sitemap/__init__.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n\n# Copyright \u00a9 2012-2014 Roberto Alsina and others.\n\n# Permission is hereby granted, free of charge, to any\n# person obtaining a copy of this software and associated\n# documentation files (the \"Software\"), to deal in the\n# Software without restriction, including without limitation\n# the rights to use, copy, modify, merge, publish,\n# distribute, sublicense, and/or sell copies of the\n# Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice\n# shall be included in all copies or substantial portions of\n# the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY\n# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE\n# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR\n# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS\n# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR\n# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\n# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\nfrom __future__ import print_function, absolute_import, unicode_literals\nimport io\nimport datetime\nimport os\ntry:\n    from urlparse import urljoin, urlparse\n    import robotparser as robotparser\nexcept ImportError:\n    from urllib.parse import urljoin, urlparse  # NOQA\n    import urllib.robotparser as robotparser  # NOQA\n\nfrom nikola.plugin_categories import LateTask\nfrom nikola.utils import config_changed\n\n\nurlset_header = \"\"\"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<urlset\n    xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n    xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9\n                        http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">\n\"\"\"\n\nloc_format = \"\"\" <url>\n  <loc>{0}</loc>\n  <lastmod>{1}</lastmod>\n </url>\n\"\"\"\n\nurlset_footer = \"</urlset>\"\n\nsitemapindex_header = \"\"\"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<sitemapindex\n    xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n    xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n    xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9\n                        http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">\n\"\"\"\n\nsitemap_format = \"\"\" <sitemap>\n  <loc>{0}</loc>\n  <lastmod>{1}</lastmod>\n </sitemap>\n\"\"\"\n\nsitemapindex_footer = \"</sitemapindex>\"\n\n\ndef get_base_path(base):\n    \"\"\"returns the path of a base URL if it contains one.\n\n    >>> get_base_path('http://some.site') == '/'\n    True\n    >>> get_base_path('http://some.site/') == '/'\n    True\n    >>> get_base_path('http://some.site/some/sub-path') == '/some/sub-path/'\n    True\n    >>> get_base_path('http://some.site/some/sub-path/') == '/some/sub-path/'\n    True\n    \"\"\"\n    # first parse the base_url for some path\n    base_parsed = urlparse(base)\n\n    if not base_parsed.path:\n        sub_path = ''\n    else:\n        sub_path = base_parsed.path\n    if sub_path.endswith('/'):\n        return sub_path\n    else:\n        return sub_path + '/'\n\n\nclass Sitemap(LateTask):\n    \"\"\"Generate a sitemap.\"\"\"\n\n    name = \"sitemap\"\n\n    def gen_tasks(self):\n        \"\"\"Generate a sitemap.\"\"\"\n        kw = {\n            \"base_url\": self.site.config[\"BASE_URL\"],\n            \"site_url\": self.site.config[\"SITE_URL\"],\n            \"output_folder\": self.site.config[\"OUTPUT_FOLDER\"],\n            \"strip_indexes\": self.site.config[\"STRIP_INDEXES\"],\n            \"index_file\": self.site.config[\"INDEX_FILE\"],\n            \"sitemap_include_fileless_dirs\": self.site.config[\"SITEMAP_INCLUDE_FILELESS_DIRS\"],\n            \"mapped_extensions\": self.site.config.get('MAPPED_EXTENSIONS', ['.html', '.htm', '.xml', '.rss']),\n            \"robots_exclusions\": self.site.config[\"ROBOTS_EXCLUSIONS\"]\n        }\n\n        output = kw['output_folder']\n        base_url = kw['base_url']\n        mapped_exts = kw['mapped_extensions']\n\n        output_path = kw['output_folder']\n        sitemapindex_path = os.path.join(output_path, \"sitemapindex.xml\")\n        sitemap_path = os.path.join(output_path, \"sitemap.xml\")\n        base_path = get_base_path(kw['base_url'])\n        sitemapindex = {}\n        urlset = {}\n\n        def scan_locs():\n            for root, dirs, files in os.walk(output, followlinks=True):\n                if not dirs and not files and not kw['sitemap_include_fileless_dirs']:\n                    continue  # Totally empty, not on sitemap\n                path = os.path.relpath(root, output)\n                # ignore the current directory.\n                path = (path.replace(os.sep, '/') + '/').replace('./', '')\n                lastmod = self.get_lastmod(root)\n                loc = urljoin(base_url, base_path + path)\n                if kw['index_file'] in files and kw['strip_indexes']:  # ignore folders when not stripping urls\n                    urlset[loc] = loc_format.format(loc, lastmod)\n                for fname in files:\n                    if kw['strip_indexes'] and fname == kw['index_file']:\n                        continue  # We already mapped the folder\n                    if os.path.splitext(fname)[-1] in mapped_exts:\n                        real_path = os.path.join(root, fname)\n                        path = os.path.relpath(real_path, output)\n                        if path.endswith(kw['index_file']) and kw['strip_indexes']:\n                            # ignore index files when stripping urls\n                            continue\n                        if not robot_fetch(path):\n                            continue\n                        if path.endswith('.html') or path.endswith('.htm'):\n                            try:\n                                if u'<!doctype html' not in io.open(real_path, 'r', encoding='utf8').read(1024).lower():\n                                    # ignores \"html\" files without doctype\n                                    # alexa-verify, google-site-verification, etc.\n                                    continue\n                            except UnicodeDecodeError:\n                                # ignore ancient files\n                                # most non-utf8 files are worthless anyways\n                                continue\n                        \"\"\" put RSS in sitemapindex[] instead of in urlset[], sitemap_path is included after it is generated \"\"\"\n                        if path.endswith('.xml') or path.endswith('.rss'):\n                            filehead = io.open(real_path, 'r', encoding='utf8').read(512)\n                            if u'<rss' in filehead or (u'<urlset' in filehead and path != sitemap_path):\n                                path = path.replace(os.sep, '/')\n                                lastmod = self.get_lastmod(real_path)\n                                loc = urljoin(base_url, base_path + path)\n                                sitemapindex[loc] = sitemap_format.format(loc, lastmod)\n                                continue\n                            else:\n                                continue  # ignores all XML files except those presumed to be RSS\n                        post = self.site.post_per_file.get(path)\n                        if post and (post.is_draft or post.is_private or post.publish_later):\n                            continue\n                        path = path.replace(os.sep, '/')\n                        lastmod = self.get_lastmod(real_path)\n                        loc = urljoin(base_url, base_path + path)\n                        urlset[loc] = loc_format.format(loc, lastmod)\n\n        def robot_fetch(path):\n            for rule in kw[\"robots_exclusions\"]:\n                robot = robotparser.RobotFileParser()\n                robot.parse([\"User-Agent: *\", \"Disallow: {0}\".format(rule)])\n                if not robot.can_fetch(\"*\", '/' + path):\n                    return False  # not robot food\n            return True\n\n        def write_sitemap():\n            # Have to rescan, because files may have been added between\n            # task dep scanning and task execution\n            with io.open(sitemap_path, 'w+', encoding='utf8') as outf:\n                outf.write(urlset_header)\n                for k in sorted(urlset.keys()):\n                    outf.write(urlset[k])\n                outf.write(urlset_footer)\n            sitemap_url = urljoin(base_url, base_path + \"sitemap.xml\")\n            sitemapindex[sitemap_url] = sitemap_format.format(sitemap_url, self.get_lastmod(sitemap_path))\n\n        def write_sitemapindex():\n            with io.open(sitemapindex_path, 'w+', encoding='utf8') as outf:\n                outf.write(sitemapindex_header)\n                for k in sorted(sitemapindex.keys()):\n                    outf.write(sitemapindex[k])\n                outf.write(sitemapindex_footer)\n\n        # Yield a task to calculate the dependencies of the sitemap\n        # Other tasks can depend on this output, instead of having\n        # to scan locations.\n        def scan_locs_task():\n            scan_locs()\n\n            # Generate a list of file dependencies for the actual generation\n            # task, so rebuilds are triggered.  (Issue #1032)\n            output = kw[\"output_folder\"]\n            file_dep = []\n\n            for i in urlset.keys():\n                p = os.path.join(output, urlparse(i).path.lstrip('/'))\n                if not p.endswith('sitemap.xml') and not p.endswith('/'):\n                    file_dep.append(p)\n                if p.endswith('/') and os.path.exists(p + 'index.html'):\n                    file_dep.append(p + 'index.html')\n\n            for i in sitemapindex.keys():\n                p = os.path.join(output, urlparse(i).path.lstrip('/'))\n                if not p.endswith('sitemap.xml') and not p.endswith('/'):\n                    file_dep.append(p)\n                if p.endswith('/') and os.path.exists(p + 'index.html'):\n                    file_dep.append(p + 'index.html')\n\n            return {'file_dep': file_dep}\n\n        yield {\n            \"basename\": \"_scan_locs\",\n            \"name\": \"sitemap\",\n            \"actions\": [(scan_locs_task)]\n        }\n\n        yield self.group_task()\n        yield {\n            \"basename\": \"sitemap\",\n            \"name\": sitemap_path,\n            \"targets\": [sitemap_path],\n            \"actions\": [(write_sitemap,)],\n            \"uptodate\": [config_changed(kw)],\n            \"clean\": True,\n            \"task_dep\": [\"render_site\"],\n            \"calc_dep\": [\"_scan_locs:sitemap\"],\n        }\n        yield {\n            \"basename\": \"sitemap\",\n            \"name\": sitemapindex_path,\n            \"targets\": [sitemapindex_path],\n            \"actions\": [(write_sitemapindex,)],\n            \"uptodate\": [config_changed(kw)],\n            \"clean\": True,\n            \"file_dep\": [sitemap_path]\n        }\n\n    def get_lastmod(self, p):\n        if self.site.invariant:\n            return '2014-01-01'\n        else:\n            return datetime.datetime.fromtimestamp(os.stat(p).st_mtime).isoformat().split('T')[0]\n\nif __name__ == '__main__':\n    import doctest\n    doctest.testmod()\n", "path": "nikola/plugins/task/sitemap/__init__.py"}]}