We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
T - Remove all non essential URLs from robots.txt
**Summary**
In the past we have added pages we didn't want to be indexed to robots.txt, but that means that Google can't crawl them to see that we don't want those pages to be indexed. We should only have pages in robots.txt that we don't want a robot to crawl (possibly due to performance issues).


**Steps To Reproduce (STR)**
1. Go to Search Console
2. go to Coverage > Valid with Warnings > Indexed, though blocked by robots.txt
3. Alternatively: https://developer.mozilla.org/robots.txt


**Actual behavior**
Google has a link to https://developer.mozilla.org/users/google/login/?next=/en-US/docs/MDN/About/Promote (for example), but that URL is blocked in robots.txt, so it can't follow the link to see that it redirects to a sign in page.


**Expected behavior**
Disallow: /*users/ should be removed from robots.txt so Google crawler can follow those urls.


**Additional context**
The reason to do this is so we can see actually problematic content show up in our search console reports, instead of this noise. Search console only shows up to 1000 pages as problematic, but there are currently more than 10k warnings, so we might be missing large issues.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `kuma/landing/views.py`
Content:
```
1 from django.conf import settings
2 from django.http import HttpResponse
3 from django.shortcuts import redirect, render
4 from django.views import static
5 from django.views.decorators.cache import never_cache
6 from django.views.generic import RedirectView
7 
8 from kuma.core.decorators import ensure_wiki_domain, shared_cache_control
9 from kuma.core.utils import is_wiki
10 from kuma.feeder.models import Bundle
11 from kuma.feeder.sections import SECTION_HACKS
12 from kuma.search.models import Filter
13 
14 from .utils import favicon_url
15 
16 
17 @shared_cache_control
18 def contribute_json(request):
19     return static.serve(request, "contribute.json", document_root=settings.ROOT)
20 
21 
22 @shared_cache_control
23 def home(request):
24     """Home page."""
25     context = {}
26     # Need for both wiki and react homepage
27     context["updates"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])
28 
29     # The default template name
30     template_name = "landing/react_homepage.html"
31     if is_wiki(request):
32         template_name = "landing/homepage.html"
33         context["default_filters"] = Filter.objects.default_filters()
34     return render(request, template_name, context)
35 
36 
37 @ensure_wiki_domain
38 @never_cache
39 def maintenance_mode(request):
40     if settings.MAINTENANCE_MODE:
41         return render(request, "landing/maintenance-mode.html")
42     else:
43         return redirect("home")
44 
45 
46 @ensure_wiki_domain
47 @shared_cache_control
48 def promote_buttons(request):
49     """Bug 646192: MDN affiliate buttons"""
50     return render(request, "landing/promote_buttons.html")
51 
52 
53 ROBOTS_ALL_ALLOWED_TXT = """\
54 User-agent: *
55 Sitemap: https://wiki.developer.mozilla.org/sitemap.xml
56 
57 Disallow:
58 """
59 
60 ROBOTS_ALLOWED_TXT = """\
61 User-agent: *
62 Sitemap: https://developer.mozilla.org/sitemap.xml
63 
64 Disallow: /api/
65 Disallow: /*docs/get-documents
66 Disallow: /*docs/Experiment:*
67 Disallow: /*$children
68 Disallow: /*docs.json
69 Disallow: /*/files/
70 Disallow: /media
71 Disallow: /*profiles*/edit
72 Disallow: /*users/
73 """ + "\n".join(
74     "Disallow: /{locale}/search".format(locale=locale)
75     for locale in settings.ENABLED_LOCALES
76 )
77 
78 ROBOTS_GO_AWAY_TXT = """\
79 User-Agent: *
80 Disallow: /
81 """
82 
83 
84 @shared_cache_control
85 def robots_txt(request):
86     """Serve robots.txt that allows or forbids robots."""
87     host = request.get_host()
88     if host in settings.ALLOW_ROBOTS_DOMAINS:
89         robots = ""
90     elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:
91         if host == settings.WIKI_HOST:
92             robots = ROBOTS_ALL_ALLOWED_TXT
93         else:
94             robots = ROBOTS_ALLOWED_TXT
95     else:
96         robots = ROBOTS_GO_AWAY_TXT
97     return HttpResponse(robots, content_type="text/plain")
98 
99 
100 class FaviconRedirect(RedirectView):
101     """Redirect to the favicon in the static img folder (bug 1402497)"""
102 
103     def get_redirect_url(self, *args, **kwargs):
104         return favicon_url()
105
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/kuma/landing/views.py b/kuma/landing/views.py
--- a/kuma/landing/views.py
+++ b/kuma/landing/views.py
@@ -69,7 +69,6 @@
 Disallow: /*/files/
 Disallow: /media
 Disallow: /*profiles*/edit
-Disallow: /*users/
 """ + "\n".join(
     "Disallow: /{locale}/search".format(locale=locale)
     for locale in settings.ENABLED_LOCALES

{"golden_diff": "diff --git a/kuma/landing/views.py b/kuma/landing/views.py\n--- a/kuma/landing/views.py\n+++ b/kuma/landing/views.py\n@@ -69,7 +69,6 @@\n Disallow: /*/files/\n Disallow: /media\n Disallow: /*profiles*/edit\n-Disallow: /*users/\n \"\"\" + \"\\n\".join(\n     \"Disallow: /{locale}/search\".format(locale=locale)\n     for locale in settings.ENABLED_LOCALES\n", "issue": "T - Remove all non essential URLs from robots.txt\n**Summary**\r\nIn the past we have added pages we didn't want to be indexed to robots.txt, but that means that Google can't crawl them to see that we don't want those pages to be indexed. We should only have pages in robots.txt that we don't want a robot to crawl (possibly due to performance issues).\r\n\r\n\r\n**Steps To Reproduce (STR)**\r\n1. Go to Search Console\r\n2. go to Coverage > Valid with Warnings > Indexed, though blocked by robots.txt\r\n3. Alternatively: https://developer.mozilla.org/robots.txt\r\n\r\n\r\n**Actual behavior**\r\nGoogle has a link to https://developer.mozilla.org/users/google/login/?next=/en-US/docs/MDN/About/Promote (for example), but that URL is blocked in robots.txt, so it can't follow the link to see that it redirects to a sign in page.\r\n\r\n\r\n**Expected behavior**\r\nDisallow: /*users/ should be removed from robots.txt so Google crawler can follow those urls.\r\n\r\n\r\n**Additional context**\r\nThe reason to do this is so we can see actually problematic content show up in our search console reports, instead of this noise. Search console only shows up to 1000 pages as problematic, but there are currently more than 10k warnings, so we might be missing large issues.\r\n\n", "before_files": [{"content": "from django.conf import settings\nfrom django.http import HttpResponse\nfrom django.shortcuts import redirect, render\nfrom django.views import static\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import RedirectView\n\nfrom kuma.core.decorators import ensure_wiki_domain, shared_cache_control\nfrom kuma.core.utils import is_wiki\nfrom kuma.feeder.models import Bundle\nfrom kuma.feeder.sections import SECTION_HACKS\nfrom kuma.search.models import Filter\n\nfrom .utils import favicon_url\n\n\n@shared_cache_control\ndef contribute_json(request):\n    return static.serve(request, \"contribute.json\", document_root=settings.ROOT)\n\n\n@shared_cache_control\ndef home(request):\n    \"\"\"Home page.\"\"\"\n    context = {}\n    # Need for both wiki and react homepage\n    context[\"updates\"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])\n\n    # The default template name\n    template_name = \"landing/react_homepage.html\"\n    if is_wiki(request):\n        template_name = \"landing/homepage.html\"\n        context[\"default_filters\"] = Filter.objects.default_filters()\n    return render(request, template_name, context)\n\n\n@ensure_wiki_domain\n@never_cache\ndef maintenance_mode(request):\n    if settings.MAINTENANCE_MODE:\n        return render(request, \"landing/maintenance-mode.html\")\n    else:\n        return redirect(\"home\")\n\n\n@ensure_wiki_domain\n@shared_cache_control\ndef promote_buttons(request):\n    \"\"\"Bug 646192: MDN affiliate buttons\"\"\"\n    return render(request, \"landing/promote_buttons.html\")\n\n\nROBOTS_ALL_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://wiki.developer.mozilla.org/sitemap.xml\n\nDisallow:\n\"\"\"\n\nROBOTS_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\n\nDisallow: /api/\nDisallow: /*docs/get-documents\nDisallow: /*docs/Experiment:*\nDisallow: /*$children\nDisallow: /*docs.json\nDisallow: /*/files/\nDisallow: /media\nDisallow: /*profiles*/edit\nDisallow: /*users/\n\"\"\" + \"\\n\".join(\n    \"Disallow: /{locale}/search\".format(locale=locale)\n    for locale in settings.ENABLED_LOCALES\n)\n\nROBOTS_GO_AWAY_TXT = \"\"\"\\\nUser-Agent: *\nDisallow: /\n\"\"\"\n\n\n@shared_cache_control\ndef robots_txt(request):\n    \"\"\"Serve robots.txt that allows or forbids robots.\"\"\"\n    host = request.get_host()\n    if host in settings.ALLOW_ROBOTS_DOMAINS:\n        robots = \"\"\n    elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:\n        if host == settings.WIKI_HOST:\n            robots = ROBOTS_ALL_ALLOWED_TXT\n        else:\n            robots = ROBOTS_ALLOWED_TXT\n    else:\n        robots = ROBOTS_GO_AWAY_TXT\n    return HttpResponse(robots, content_type=\"text/plain\")\n\n\nclass FaviconRedirect(RedirectView):\n    \"\"\"Redirect to the favicon in the static img folder (bug 1402497)\"\"\"\n\n    def get_redirect_url(self, *args, **kwargs):\n        return favicon_url()\n", "path": "kuma/landing/views.py"}], "after_files": [{"content": "from django.conf import settings\nfrom django.http import HttpResponse\nfrom django.shortcuts import redirect, render\nfrom django.views import static\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import RedirectView\n\nfrom kuma.core.decorators import ensure_wiki_domain, shared_cache_control\nfrom kuma.core.utils import is_wiki\nfrom kuma.feeder.models import Bundle\nfrom kuma.feeder.sections import SECTION_HACKS\nfrom kuma.search.models import Filter\n\nfrom .utils import favicon_url\n\n\n@shared_cache_control\ndef contribute_json(request):\n    return static.serve(request, \"contribute.json\", document_root=settings.ROOT)\n\n\n@shared_cache_control\ndef home(request):\n    \"\"\"Home page.\"\"\"\n    context = {}\n    # Need for both wiki and react homepage\n    context[\"updates\"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])\n\n    # The default template name\n    template_name = \"landing/react_homepage.html\"\n    if is_wiki(request):\n        template_name = \"landing/homepage.html\"\n        context[\"default_filters\"] = Filter.objects.default_filters()\n    return render(request, template_name, context)\n\n\n@ensure_wiki_domain\n@never_cache\ndef maintenance_mode(request):\n    if settings.MAINTENANCE_MODE:\n        return render(request, \"landing/maintenance-mode.html\")\n    else:\n        return redirect(\"home\")\n\n\n@ensure_wiki_domain\n@shared_cache_control\ndef promote_buttons(request):\n    \"\"\"Bug 646192: MDN affiliate buttons\"\"\"\n    return render(request, \"landing/promote_buttons.html\")\n\n\nROBOTS_ALL_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://wiki.developer.mozilla.org/sitemap.xml\n\nDisallow:\n\"\"\"\n\nROBOTS_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\n\nDisallow: /api/\nDisallow: /*docs/get-documents\nDisallow: /*docs/Experiment:*\nDisallow: /*$children\nDisallow: /*docs.json\nDisallow: /*/files/\nDisallow: /media\nDisallow: /*profiles*/edit\n\"\"\" + \"\\n\".join(\n    \"Disallow: /{locale}/search\".format(locale=locale)\n    for locale in settings.ENABLED_LOCALES\n)\n\nROBOTS_GO_AWAY_TXT = \"\"\"\\\nUser-Agent: *\nDisallow: /\n\"\"\"\n\n\n@shared_cache_control\ndef robots_txt(request):\n    \"\"\"Serve robots.txt that allows or forbids robots.\"\"\"\n    host = request.get_host()\n    if host in settings.ALLOW_ROBOTS_DOMAINS:\n        robots = \"\"\n    elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:\n        if host == settings.WIKI_HOST:\n            robots = ROBOTS_ALL_ALLOWED_TXT\n        else:\n            robots = ROBOTS_ALLOWED_TXT\n    else:\n        robots = ROBOTS_GO_AWAY_TXT\n    return HttpResponse(robots, content_type=\"text/plain\")\n\n\nclass FaviconRedirect(RedirectView):\n    \"\"\"Redirect to the favicon in the static img folder (bug 1402497)\"\"\"\n\n    def get_redirect_url(self, *args, **kwargs):\n        return favicon_url()\n", "path": "kuma/landing/views.py"}]}