We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
mypy plugin fails; site dir is in PYTHONPATH
I've been trying to update from pants 1.21.0 to 1.24.0 .

When I run this command (on both mac and linux)
`./pants lint.mypy tool/::`
I get an error just on the newer version of pants:
```
/Users/me/my_buildroot is in the PYTHONPATH. Please change directory so it is not.
```

This happens even if I pin mypy to 0.710, the version we were defaulting to with pants 1.21.0 .

You can seen here that mypy doesn't want a folder to be both in PYTHONPATH and site packages:
https://github.com/python/mypy/blob/v0.710/mypy/modulefinder.py#L488

One potentially unique feature of our repo is that we set `pythonpath` in pants.ini to the root of our repo.

```
pythonpath: +[
    "%(buildroot)s",
  ]
backend_packages: +[
    "package.plugin1",
    "package.plugin2",
  ]
```

However, I tried changing this global pythonpath config (which I think should only affect the python environment in which pants itself is running) and it didn't seem to help.




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py`
Content:
```
1 # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
2 # Licensed under the Apache License, Version 2.0 (see LICENSE).
3 
4 import os
5 from pathlib import Path
6 from textwrap import dedent
7 from typing import Iterable, List, Set
8 
9 from pants.backend.python.interpreter_cache import PythonInterpreterCache
10 from pants.backend.python.targets.python_binary import PythonBinary
11 from pants.backend.python.targets.python_library import PythonLibrary
12 from pants.backend.python.targets.python_target import PythonTarget
13 from pants.backend.python.targets.python_tests import PythonTests
14 from pants.backend.python.tasks.resolve_requirements import ResolveRequirements
15 from pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase
16 from pants.base import hash_utils
17 from pants.base.build_environment import get_buildroot
18 from pants.base.exceptions import TaskError
19 from pants.base.workunit import WorkUnitLabel
20 from pants.build_graph.target import Target
21 from pants.task.lint_task_mixin import LintTaskMixin
22 from pants.util.contextutil import temporary_file, temporary_file_path
23 from pants.util.memo import memoized_property
24 from pex.interpreter import PythonInterpreter
25 from pex.pex import PEX
26 from pex.pex_info import PexInfo
27 
28 from pants.contrib.mypy.subsystems.subsystem import MyPy
29 
30 
31 class MypyTaskError(TaskError):
32     """Indicates a TaskError from a failing MyPy run."""
33 
34 
35 class MypyTask(LintTaskMixin, ResolveRequirementsTaskBase):
36     """Invoke the mypy static type analyzer for Python.
37 
38     Mypy lint task filters out target_roots that are not properly tagged according to
39     --whitelisted-tag-name (defaults to None, and no filtering occurs if this option is 'None'),
40     and executes MyPy on targets in context from whitelisted target roots.
41     (if any transitive targets from the filtered roots are not whitelisted, a warning
42     will be printed.)
43 
44     'In context' meaning in the sub-graph where a whitelisted target is the root
45     """
46 
47     _MYPY_COMPATIBLE_INTERPETER_CONSTRAINT = ">=3.5"
48     _PYTHON_SOURCE_EXTENSION = ".py"
49 
50     @classmethod
51     def prepare(cls, options, round_manager):
52         super().prepare(options, round_manager)
53         round_manager.require_data(PythonInterpreter)
54         if options.include_requirements:
55             round_manager.require_data(ResolveRequirements.REQUIREMENTS_PEX)
56 
57     @classmethod
58     def register_options(cls, register):
59         register(
60             "--include-requirements",
61             type=bool,
62             default=False,
63             help="Whether to include the transitive requirements of targets being checked. This is"
64             "useful if those targets depend on mypy plugins or distributions that provide "
65             "type stubs that should be active in the check.",
66         )
67         register(
68             "--whitelist-tag-name",
69             default=None,
70             help="Tag name to identify Python targets to execute MyPy",
71         )
72         register(
73             "--verbose",
74             type=bool,
75             default=False,
76             help="Extra detail showing non-whitelisted targets",
77         )
78 
79     @classmethod
80     def subsystem_dependencies(cls):
81         return super().subsystem_dependencies() + (PythonInterpreterCache, MyPy)
82 
83     @property
84     def skip_execution(self):
85         return self._mypy_subsystem.options.skip
86 
87     def find_mypy_interpreter(self):
88         interpreters = self._interpreter_cache.setup(
89             filters=[self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT]
90         )
91         return min(interpreters) if interpreters else None
92 
93     @staticmethod
94     def is_non_synthetic_python_target(target):
95         return not target.is_synthetic and isinstance(
96             target, (PythonLibrary, PythonBinary, PythonTests)
97         )
98 
99     @staticmethod
100     def is_python_target(target):
101         return isinstance(target, PythonTarget)
102 
103     def _check_for_untagged_dependencies(
104         self, *, tagged_target_roots: Iterable[Target], tag_name: str
105     ) -> None:
106         untagged_dependencies = {
107             tgt
108             for tgt in Target.closure_for_targets(target_roots=tagged_target_roots)
109             if tag_name not in tgt.tags and self.is_non_synthetic_python_target(tgt)
110         }
111         if not untagged_dependencies:
112             return
113         formatted_targets = "\n".join(tgt.address.spec for tgt in sorted(untagged_dependencies))
114         self.context.log.warn(
115             f"[WARNING]: The following targets are not marked with the tag name `{tag_name}`, "
116             f"but are dependencies of targets that are type checked. MyPy will check these dependencies, "
117             f"inferring `Any` where possible. You are encouraged to properly type check "
118             f"these dependencies.\n{formatted_targets}"
119         )
120 
121     def _calculate_python_sources(self, target_roots: Iterable[Target]) -> List[str]:
122         """Filter targets to generate a set of source files from the given targets."""
123         all_targets = {
124             tgt
125             for tgt in Target.closure_for_targets(target_roots=target_roots)
126             if self.is_non_synthetic_python_target(tgt)
127         }
128         whitelist_tag_name = self.get_options().whitelist_tag_name
129         if whitelist_tag_name:
130             tagged_targets = {tgt for tgt in all_targets if whitelist_tag_name in tgt.tags}
131             eval_targets = tagged_targets
132             if self.get_options().verbose:
133                 self._check_for_untagged_dependencies(
134                     tagged_target_roots={tgt for tgt in tagged_targets if tgt in target_roots},
135                     tag_name=whitelist_tag_name,
136                 )
137         else:
138             eval_targets = all_targets
139 
140         sources: Set[str] = set()
141         for target in eval_targets:
142             sources.update(
143                 source
144                 for source in target.sources_relative_to_buildroot()
145                 if os.path.splitext(source)[1] == self._PYTHON_SOURCE_EXTENSION
146             )
147         return list(sorted(sources))
148 
149     def _collect_source_roots(self):
150         # Collect the set of directories in which there are Python sources (whether part of
151         # the target roots or transitive dependencies.)
152         source_roots = set()
153         for target in self.context.targets(self.is_python_target):
154             if not target.has_sources(self._PYTHON_SOURCE_EXTENSION):
155                 continue
156             source_roots.add(target.target_base)
157         return source_roots
158 
159     @memoized_property
160     def _interpreter_cache(self):
161         return PythonInterpreterCache.global_instance()
162 
163     @memoized_property
164     def _mypy_subsystem(self):
165         return MyPy.global_instance()
166 
167     def _get_mypy_pex(self, py3_interpreter: PythonInterpreter, *extra_pexes: PEX) -> PEX:
168         mypy_version = self._mypy_subsystem.options.version
169         extras_hash = hash_utils.hash_all(
170             hash_utils.hash_dir(Path(extra_pex.path())) for extra_pex in extra_pexes
171         )
172 
173         path = Path(self.workdir, str(py3_interpreter.identity), f"{mypy_version}-{extras_hash}")
174         pex_dir = str(path)
175         if not path.is_dir():
176             mypy_requirement_pex = self.resolve_requirement_strings(py3_interpreter, [mypy_version])
177             pex_info = PexInfo.default()
178             pex_info.entry_point = "pants_mypy_launcher"
179             with self.merged_pex(
180                 path=pex_dir,
181                 pex_info=pex_info,
182                 interpreter=py3_interpreter,
183                 pexes=[mypy_requirement_pex, *extra_pexes],
184             ) as builder:
185                 with temporary_file(binary_mode=False) as exe_fp:
186                     # MyPy searches for types for a package in packages containing a `py.types` marker file
187                     # or else in a sibling `<package>-stubs` package as per PEP-0561. Going further than that
188                     # PEP, MyPy restricts its search to `site-packages`. Since PEX deliberately isolates
189                     # itself from `site-packages` as part of its raison d'etre, we monkey-patch
190                     # `site.getsitepackages` to look inside the scrubbed PEX sys.path before handing off to
191                     # `mypy`.
192                     #
193                     # See:
194                     #   https://mypy.readthedocs.io/en/stable/installed_packages.html#installed-packages
195                     #   https://www.python.org/dev/peps/pep-0561/#stub-only-packages
196                     exe_fp.write(
197                         dedent(
198                             """
199                             import runpy
200                             import site
201                             import sys
202                 
203                 
204                             site.getsitepackages = lambda: sys.path[:]
205                 
206                             
207                             runpy.run_module('mypy', run_name='__main__')
208                             """
209                         )
210                     )
211                     exe_fp.flush()
212                     builder.set_executable(
213                         filename=exe_fp.name, env_filename=f"{pex_info.entry_point}.py"
214                     )
215                 builder.freeze(bytecode_compile=False)
216 
217         return PEX(pex_dir, py3_interpreter)
218 
219     def execute(self):
220         mypy_interpreter = self.find_mypy_interpreter()
221         if not mypy_interpreter:
222             raise TaskError(
223                 f"Unable to find a Python {self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT} "
224                 f"interpreter (required for mypy)."
225             )
226 
227         sources = self._calculate_python_sources(self.context.target_roots)
228         if not sources:
229             self.context.log.debug("No Python sources to check.")
230             return
231 
232         # Determine interpreter used by the sources so we can tell mypy.
233         interpreter_for_targets = self._interpreter_cache.select_interpreter_for_targets(
234             self.context.target_roots
235         )
236         if not interpreter_for_targets:
237             raise TaskError("No Python interpreter compatible with specified sources.")
238 
239         extra_pexes = []
240         if self.get_options().include_requirements:
241             if interpreter_for_targets.identity.matches(
242                 self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT
243             ):
244                 extra_pexes.append(
245                     self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
246                 )
247                 mypy_interpreter = interpreter_for_targets
248             else:
249                 self.context.log.warn(
250                     f"The --include-requirements option is set, but the current target's requirements have "
251                     f"been resolved for {interpreter_for_targets.identity} which is not compatible with mypy "
252                     f"which needs {self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT}: omitting resolved "
253                     f"requirements from the mypy PYTHONPATH."
254                 )
255 
256         with temporary_file_path() as sources_list_path:
257             with open(sources_list_path, "w") as f:
258                 for source in sources:
259                     f.write(f"{source}\n")
260             # Construct the mypy command line.
261             cmd = [f"--python-version={interpreter_for_targets.identity.python}"]
262 
263             config = self._mypy_subsystem.options.config
264             if config:
265                 cmd.append(f"--config-file={os.path.join(get_buildroot(), config)}")
266             cmd.extend(self._mypy_subsystem.options.args)
267             cmd.append(f"@{sources_list_path}")
268 
269             with self.context.new_workunit(name="create_mypy_pex", labels=[WorkUnitLabel.PREP]):
270                 mypy_pex = self._get_mypy_pex(mypy_interpreter, *extra_pexes)
271 
272             # Collect source roots for the targets being checked.
273             buildroot = Path(get_buildroot())
274             sources_path = os.pathsep.join(
275                 str(buildroot.joinpath(root)) for root in self._collect_source_roots()
276             )
277 
278             # Execute mypy.
279             with self.context.new_workunit(
280                 name="check",
281                 labels=[WorkUnitLabel.TOOL, WorkUnitLabel.RUN],
282                 cmd=" ".join(mypy_pex.cmdline(cmd)),
283             ) as workunit:
284                 returncode = mypy_pex.run(
285                     cmd,
286                     env=dict(PYTHONPATH=sources_path, PEX_INHERIT_PATH="fallback"),
287                     stdout=workunit.output("stdout"),
288                     stderr=workunit.output("stderr"),
289                 )
290                 if returncode != 0:
291                     raise MypyTaskError(f"mypy failed: code={returncode}")
292
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py b/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py
--- a/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py
+++ b/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py
@@ -183,12 +183,17 @@
                 pexes=[mypy_requirement_pex, *extra_pexes],
             ) as builder:
                 with temporary_file(binary_mode=False) as exe_fp:
-                    # MyPy searches for types for a package in packages containing a `py.types` marker file
-                    # or else in a sibling `<package>-stubs` package as per PEP-0561. Going further than that
-                    # PEP, MyPy restricts its search to `site-packages`. Since PEX deliberately isolates
-                    # itself from `site-packages` as part of its raison d'etre, we monkey-patch
-                    # `site.getsitepackages` to look inside the scrubbed PEX sys.path before handing off to
-                    # `mypy`.
+                    # MyPy searches for types for a package in packages containing a `py.types`
+                    # marker file or else in a sibling `<package>-stubs` package as per PEP-0561.
+                    # Going further than that PEP, MyPy restricts its search to `site-packages`.
+                    # Since PEX deliberately isolates itself from `site-packages` as part of its
+                    # raison d'etre, we monkey-patch `site.getsitepackages` to look inside the
+                    # scrubbed PEX sys.path before handing off to `mypy`.
+                    #
+                    # As a complication, MyPy does its own validation to ensure packages aren't
+                    # both available in site-packages and on the PYTHONPATH. As such, we elide all
+                    # PYTHONPATH entries from artificial site-packages we set up since MyPy will
+                    # manually scan PYTHONPATH outside this PEX to find packages.
                     #
                     # See:
                     #   https://mypy.readthedocs.io/en/stable/installed_packages.html#installed-packages
@@ -196,14 +201,20 @@
                     exe_fp.write(
                         dedent(
                             """
+                            import os
                             import runpy
                             import site
                             import sys
-                
-                
-                            site.getsitepackages = lambda: sys.path[:]
-                
+
+                            PYTHONPATH = frozenset(
+                                os.path.realpath(p)
+                                for p in os.environ.get('PYTHONPATH', '').split(os.pathsep)
+                            )
                             
+                            site.getsitepackages = lambda: [
+                                p for p in sys.path if os.path.realpath(p) not in PYTHONPATH
+                            ]
+
                             runpy.run_module('mypy', run_name='__main__')
                             """
                         )

{"golden_diff": "diff --git a/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py b/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py\n--- a/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py\n+++ b/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py\n@@ -183,12 +183,17 @@\n                 pexes=[mypy_requirement_pex, *extra_pexes],\n             ) as builder:\n                 with temporary_file(binary_mode=False) as exe_fp:\n-                    # MyPy searches for types for a package in packages containing a `py.types` marker file\n-                    # or else in a sibling `<package>-stubs` package as per PEP-0561. Going further than that\n-                    # PEP, MyPy restricts its search to `site-packages`. Since PEX deliberately isolates\n-                    # itself from `site-packages` as part of its raison d'etre, we monkey-patch\n-                    # `site.getsitepackages` to look inside the scrubbed PEX sys.path before handing off to\n-                    # `mypy`.\n+                    # MyPy searches for types for a package in packages containing a `py.types`\n+                    # marker file or else in a sibling `<package>-stubs` package as per PEP-0561.\n+                    # Going further than that PEP, MyPy restricts its search to `site-packages`.\n+                    # Since PEX deliberately isolates itself from `site-packages` as part of its\n+                    # raison d'etre, we monkey-patch `site.getsitepackages` to look inside the\n+                    # scrubbed PEX sys.path before handing off to `mypy`.\n+                    #\n+                    # As a complication, MyPy does its own validation to ensure packages aren't\n+                    # both available in site-packages and on the PYTHONPATH. As such, we elide all\n+                    # PYTHONPATH entries from artificial site-packages we set up since MyPy will\n+                    # manually scan PYTHONPATH outside this PEX to find packages.\n                     #\n                     # See:\n                     #   https://mypy.readthedocs.io/en/stable/installed_packages.html#installed-packages\n@@ -196,14 +201,20 @@\n                     exe_fp.write(\n                         dedent(\n                             \"\"\"\n+                            import os\n                             import runpy\n                             import site\n                             import sys\n-                \n-                \n-                            site.getsitepackages = lambda: sys.path[:]\n-                \n+\n+                            PYTHONPATH = frozenset(\n+                                os.path.realpath(p)\n+                                for p in os.environ.get('PYTHONPATH', '').split(os.pathsep)\n+                            )\n                             \n+                            site.getsitepackages = lambda: [\n+                                p for p in sys.path if os.path.realpath(p) not in PYTHONPATH\n+                            ]\n+\n                             runpy.run_module('mypy', run_name='__main__')\n                             \"\"\"\n                         )\n", "issue": "mypy plugin fails; site dir is in PYTHONPATH\nI've been trying to update from pants 1.21.0 to 1.24.0 .\r\n\r\nWhen I run this command (on both mac and linux)\r\n`./pants lint.mypy tool/::`\r\nI get an error just on the newer version of pants:\r\n```\r\n/Users/me/my_buildroot is in the PYTHONPATH. Please change directory so it is not.\r\n```\r\n\r\nThis happens even if I pin mypy to 0.710, the version we were defaulting to with pants 1.21.0 .\r\n\r\nYou can seen here that mypy doesn't want a folder to be both in PYTHONPATH and site packages:\r\nhttps://github.com/python/mypy/blob/v0.710/mypy/modulefinder.py#L488\r\n\r\nOne potentially unique feature of our repo is that we set `pythonpath` in pants.ini to the root of our repo.\r\n\r\n```\r\npythonpath: +[\r\n    \"%(buildroot)s\",\r\n  ]\r\nbackend_packages: +[\r\n    \"package.plugin1\",\r\n    \"package.plugin2\",\r\n  ]\r\n```\r\n\r\nHowever, I tried changing this global pythonpath config (which I think should only affect the python environment in which pants itself is running) and it didn't seem to help.\r\n\r\n\r\n\n", "before_files": [{"content": "# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).\n# Licensed under the Apache License, Version 2.0 (see LICENSE).\n\nimport os\nfrom pathlib import Path\nfrom textwrap import dedent\nfrom typing import Iterable, List, Set\n\nfrom pants.backend.python.interpreter_cache import PythonInterpreterCache\nfrom pants.backend.python.targets.python_binary import PythonBinary\nfrom pants.backend.python.targets.python_library import PythonLibrary\nfrom pants.backend.python.targets.python_target import PythonTarget\nfrom pants.backend.python.targets.python_tests import PythonTests\nfrom pants.backend.python.tasks.resolve_requirements import ResolveRequirements\nfrom pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase\nfrom pants.base import hash_utils\nfrom pants.base.build_environment import get_buildroot\nfrom pants.base.exceptions import TaskError\nfrom pants.base.workunit import WorkUnitLabel\nfrom pants.build_graph.target import Target\nfrom pants.task.lint_task_mixin import LintTaskMixin\nfrom pants.util.contextutil import temporary_file, temporary_file_path\nfrom pants.util.memo import memoized_property\nfrom pex.interpreter import PythonInterpreter\nfrom pex.pex import PEX\nfrom pex.pex_info import PexInfo\n\nfrom pants.contrib.mypy.subsystems.subsystem import MyPy\n\n\nclass MypyTaskError(TaskError):\n    \"\"\"Indicates a TaskError from a failing MyPy run.\"\"\"\n\n\nclass MypyTask(LintTaskMixin, ResolveRequirementsTaskBase):\n    \"\"\"Invoke the mypy static type analyzer for Python.\n\n    Mypy lint task filters out target_roots that are not properly tagged according to\n    --whitelisted-tag-name (defaults to None, and no filtering occurs if this option is 'None'),\n    and executes MyPy on targets in context from whitelisted target roots.\n    (if any transitive targets from the filtered roots are not whitelisted, a warning\n    will be printed.)\n\n    'In context' meaning in the sub-graph where a whitelisted target is the root\n    \"\"\"\n\n    _MYPY_COMPATIBLE_INTERPETER_CONSTRAINT = \">=3.5\"\n    _PYTHON_SOURCE_EXTENSION = \".py\"\n\n    @classmethod\n    def prepare(cls, options, round_manager):\n        super().prepare(options, round_manager)\n        round_manager.require_data(PythonInterpreter)\n        if options.include_requirements:\n            round_manager.require_data(ResolveRequirements.REQUIREMENTS_PEX)\n\n    @classmethod\n    def register_options(cls, register):\n        register(\n            \"--include-requirements\",\n            type=bool,\n            default=False,\n            help=\"Whether to include the transitive requirements of targets being checked. This is\"\n            \"useful if those targets depend on mypy plugins or distributions that provide \"\n            \"type stubs that should be active in the check.\",\n        )\n        register(\n            \"--whitelist-tag-name\",\n            default=None,\n            help=\"Tag name to identify Python targets to execute MyPy\",\n        )\n        register(\n            \"--verbose\",\n            type=bool,\n            default=False,\n            help=\"Extra detail showing non-whitelisted targets\",\n        )\n\n    @classmethod\n    def subsystem_dependencies(cls):\n        return super().subsystem_dependencies() + (PythonInterpreterCache, MyPy)\n\n    @property\n    def skip_execution(self):\n        return self._mypy_subsystem.options.skip\n\n    def find_mypy_interpreter(self):\n        interpreters = self._interpreter_cache.setup(\n            filters=[self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT]\n        )\n        return min(interpreters) if interpreters else None\n\n    @staticmethod\n    def is_non_synthetic_python_target(target):\n        return not target.is_synthetic and isinstance(\n            target, (PythonLibrary, PythonBinary, PythonTests)\n        )\n\n    @staticmethod\n    def is_python_target(target):\n        return isinstance(target, PythonTarget)\n\n    def _check_for_untagged_dependencies(\n        self, *, tagged_target_roots: Iterable[Target], tag_name: str\n    ) -> None:\n        untagged_dependencies = {\n            tgt\n            for tgt in Target.closure_for_targets(target_roots=tagged_target_roots)\n            if tag_name not in tgt.tags and self.is_non_synthetic_python_target(tgt)\n        }\n        if not untagged_dependencies:\n            return\n        formatted_targets = \"\\n\".join(tgt.address.spec for tgt in sorted(untagged_dependencies))\n        self.context.log.warn(\n            f\"[WARNING]: The following targets are not marked with the tag name `{tag_name}`, \"\n            f\"but are dependencies of targets that are type checked. MyPy will check these dependencies, \"\n            f\"inferring `Any` where possible. You are encouraged to properly type check \"\n            f\"these dependencies.\\n{formatted_targets}\"\n        )\n\n    def _calculate_python_sources(self, target_roots: Iterable[Target]) -> List[str]:\n        \"\"\"Filter targets to generate a set of source files from the given targets.\"\"\"\n        all_targets = {\n            tgt\n            for tgt in Target.closure_for_targets(target_roots=target_roots)\n            if self.is_non_synthetic_python_target(tgt)\n        }\n        whitelist_tag_name = self.get_options().whitelist_tag_name\n        if whitelist_tag_name:\n            tagged_targets = {tgt for tgt in all_targets if whitelist_tag_name in tgt.tags}\n            eval_targets = tagged_targets\n            if self.get_options().verbose:\n                self._check_for_untagged_dependencies(\n                    tagged_target_roots={tgt for tgt in tagged_targets if tgt in target_roots},\n                    tag_name=whitelist_tag_name,\n                )\n        else:\n            eval_targets = all_targets\n\n        sources: Set[str] = set()\n        for target in eval_targets:\n            sources.update(\n                source\n                for source in target.sources_relative_to_buildroot()\n                if os.path.splitext(source)[1] == self._PYTHON_SOURCE_EXTENSION\n            )\n        return list(sorted(sources))\n\n    def _collect_source_roots(self):\n        # Collect the set of directories in which there are Python sources (whether part of\n        # the target roots or transitive dependencies.)\n        source_roots = set()\n        for target in self.context.targets(self.is_python_target):\n            if not target.has_sources(self._PYTHON_SOURCE_EXTENSION):\n                continue\n            source_roots.add(target.target_base)\n        return source_roots\n\n    @memoized_property\n    def _interpreter_cache(self):\n        return PythonInterpreterCache.global_instance()\n\n    @memoized_property\n    def _mypy_subsystem(self):\n        return MyPy.global_instance()\n\n    def _get_mypy_pex(self, py3_interpreter: PythonInterpreter, *extra_pexes: PEX) -> PEX:\n        mypy_version = self._mypy_subsystem.options.version\n        extras_hash = hash_utils.hash_all(\n            hash_utils.hash_dir(Path(extra_pex.path())) for extra_pex in extra_pexes\n        )\n\n        path = Path(self.workdir, str(py3_interpreter.identity), f\"{mypy_version}-{extras_hash}\")\n        pex_dir = str(path)\n        if not path.is_dir():\n            mypy_requirement_pex = self.resolve_requirement_strings(py3_interpreter, [mypy_version])\n            pex_info = PexInfo.default()\n            pex_info.entry_point = \"pants_mypy_launcher\"\n            with self.merged_pex(\n                path=pex_dir,\n                pex_info=pex_info,\n                interpreter=py3_interpreter,\n                pexes=[mypy_requirement_pex, *extra_pexes],\n            ) as builder:\n                with temporary_file(binary_mode=False) as exe_fp:\n                    # MyPy searches for types for a package in packages containing a `py.types` marker file\n                    # or else in a sibling `<package>-stubs` package as per PEP-0561. Going further than that\n                    # PEP, MyPy restricts its search to `site-packages`. Since PEX deliberately isolates\n                    # itself from `site-packages` as part of its raison d'etre, we monkey-patch\n                    # `site.getsitepackages` to look inside the scrubbed PEX sys.path before handing off to\n                    # `mypy`.\n                    #\n                    # See:\n                    #   https://mypy.readthedocs.io/en/stable/installed_packages.html#installed-packages\n                    #   https://www.python.org/dev/peps/pep-0561/#stub-only-packages\n                    exe_fp.write(\n                        dedent(\n                            \"\"\"\n                            import runpy\n                            import site\n                            import sys\n                \n                \n                            site.getsitepackages = lambda: sys.path[:]\n                \n                            \n                            runpy.run_module('mypy', run_name='__main__')\n                            \"\"\"\n                        )\n                    )\n                    exe_fp.flush()\n                    builder.set_executable(\n                        filename=exe_fp.name, env_filename=f\"{pex_info.entry_point}.py\"\n                    )\n                builder.freeze(bytecode_compile=False)\n\n        return PEX(pex_dir, py3_interpreter)\n\n    def execute(self):\n        mypy_interpreter = self.find_mypy_interpreter()\n        if not mypy_interpreter:\n            raise TaskError(\n                f\"Unable to find a Python {self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT} \"\n                f\"interpreter (required for mypy).\"\n            )\n\n        sources = self._calculate_python_sources(self.context.target_roots)\n        if not sources:\n            self.context.log.debug(\"No Python sources to check.\")\n            return\n\n        # Determine interpreter used by the sources so we can tell mypy.\n        interpreter_for_targets = self._interpreter_cache.select_interpreter_for_targets(\n            self.context.target_roots\n        )\n        if not interpreter_for_targets:\n            raise TaskError(\"No Python interpreter compatible with specified sources.\")\n\n        extra_pexes = []\n        if self.get_options().include_requirements:\n            if interpreter_for_targets.identity.matches(\n                self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT\n            ):\n                extra_pexes.append(\n                    self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)\n                )\n                mypy_interpreter = interpreter_for_targets\n            else:\n                self.context.log.warn(\n                    f\"The --include-requirements option is set, but the current target's requirements have \"\n                    f\"been resolved for {interpreter_for_targets.identity} which is not compatible with mypy \"\n                    f\"which needs {self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT}: omitting resolved \"\n                    f\"requirements from the mypy PYTHONPATH.\"\n                )\n\n        with temporary_file_path() as sources_list_path:\n            with open(sources_list_path, \"w\") as f:\n                for source in sources:\n                    f.write(f\"{source}\\n\")\n            # Construct the mypy command line.\n            cmd = [f\"--python-version={interpreter_for_targets.identity.python}\"]\n\n            config = self._mypy_subsystem.options.config\n            if config:\n                cmd.append(f\"--config-file={os.path.join(get_buildroot(), config)}\")\n            cmd.extend(self._mypy_subsystem.options.args)\n            cmd.append(f\"@{sources_list_path}\")\n\n            with self.context.new_workunit(name=\"create_mypy_pex\", labels=[WorkUnitLabel.PREP]):\n                mypy_pex = self._get_mypy_pex(mypy_interpreter, *extra_pexes)\n\n            # Collect source roots for the targets being checked.\n            buildroot = Path(get_buildroot())\n            sources_path = os.pathsep.join(\n                str(buildroot.joinpath(root)) for root in self._collect_source_roots()\n            )\n\n            # Execute mypy.\n            with self.context.new_workunit(\n                name=\"check\",\n                labels=[WorkUnitLabel.TOOL, WorkUnitLabel.RUN],\n                cmd=\" \".join(mypy_pex.cmdline(cmd)),\n            ) as workunit:\n                returncode = mypy_pex.run(\n                    cmd,\n                    env=dict(PYTHONPATH=sources_path, PEX_INHERIT_PATH=\"fallback\"),\n                    stdout=workunit.output(\"stdout\"),\n                    stderr=workunit.output(\"stderr\"),\n                )\n                if returncode != 0:\n                    raise MypyTaskError(f\"mypy failed: code={returncode}\")\n", "path": "contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py"}], "after_files": [{"content": "# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).\n# Licensed under the Apache License, Version 2.0 (see LICENSE).\n\nimport os\nfrom pathlib import Path\nfrom textwrap import dedent\nfrom typing import Iterable, List, Set\n\nfrom pants.backend.python.interpreter_cache import PythonInterpreterCache\nfrom pants.backend.python.targets.python_binary import PythonBinary\nfrom pants.backend.python.targets.python_library import PythonLibrary\nfrom pants.backend.python.targets.python_target import PythonTarget\nfrom pants.backend.python.targets.python_tests import PythonTests\nfrom pants.backend.python.tasks.resolve_requirements import ResolveRequirements\nfrom pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase\nfrom pants.base import hash_utils\nfrom pants.base.build_environment import get_buildroot\nfrom pants.base.exceptions import TaskError\nfrom pants.base.workunit import WorkUnitLabel\nfrom pants.build_graph.target import Target\nfrom pants.task.lint_task_mixin import LintTaskMixin\nfrom pants.util.contextutil import temporary_file, temporary_file_path\nfrom pants.util.memo import memoized_property\nfrom pex.interpreter import PythonInterpreter\nfrom pex.pex import PEX\nfrom pex.pex_info import PexInfo\n\nfrom pants.contrib.mypy.subsystems.subsystem import MyPy\n\n\nclass MypyTaskError(TaskError):\n    \"\"\"Indicates a TaskError from a failing MyPy run.\"\"\"\n\n\nclass MypyTask(LintTaskMixin, ResolveRequirementsTaskBase):\n    \"\"\"Invoke the mypy static type analyzer for Python.\n\n    Mypy lint task filters out target_roots that are not properly tagged according to\n    --whitelisted-tag-name (defaults to None, and no filtering occurs if this option is 'None'),\n    and executes MyPy on targets in context from whitelisted target roots.\n    (if any transitive targets from the filtered roots are not whitelisted, a warning\n    will be printed.)\n\n    'In context' meaning in the sub-graph where a whitelisted target is the root\n    \"\"\"\n\n    _MYPY_COMPATIBLE_INTERPETER_CONSTRAINT = \">=3.5\"\n    _PYTHON_SOURCE_EXTENSION = \".py\"\n\n    @classmethod\n    def prepare(cls, options, round_manager):\n        super().prepare(options, round_manager)\n        round_manager.require_data(PythonInterpreter)\n        if options.include_requirements:\n            round_manager.require_data(ResolveRequirements.REQUIREMENTS_PEX)\n\n    @classmethod\n    def register_options(cls, register):\n        register(\n            \"--include-requirements\",\n            type=bool,\n            default=False,\n            help=\"Whether to include the transitive requirements of targets being checked. This is\"\n            \"useful if those targets depend on mypy plugins or distributions that provide \"\n            \"type stubs that should be active in the check.\",\n        )\n        register(\n            \"--whitelist-tag-name\",\n            default=None,\n            help=\"Tag name to identify Python targets to execute MyPy\",\n        )\n        register(\n            \"--verbose\",\n            type=bool,\n            default=False,\n            help=\"Extra detail showing non-whitelisted targets\",\n        )\n\n    @classmethod\n    def subsystem_dependencies(cls):\n        return super().subsystem_dependencies() + (PythonInterpreterCache, MyPy)\n\n    @property\n    def skip_execution(self):\n        return self._mypy_subsystem.options.skip\n\n    def find_mypy_interpreter(self):\n        interpreters = self._interpreter_cache.setup(\n            filters=[self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT]\n        )\n        return min(interpreters) if interpreters else None\n\n    @staticmethod\n    def is_non_synthetic_python_target(target):\n        return not target.is_synthetic and isinstance(\n            target, (PythonLibrary, PythonBinary, PythonTests)\n        )\n\n    @staticmethod\n    def is_python_target(target):\n        return isinstance(target, PythonTarget)\n\n    def _check_for_untagged_dependencies(\n        self, *, tagged_target_roots: Iterable[Target], tag_name: str\n    ) -> None:\n        untagged_dependencies = {\n            tgt\n            for tgt in Target.closure_for_targets(target_roots=tagged_target_roots)\n            if tag_name not in tgt.tags and self.is_non_synthetic_python_target(tgt)\n        }\n        if not untagged_dependencies:\n            return\n        formatted_targets = \"\\n\".join(tgt.address.spec for tgt in sorted(untagged_dependencies))\n        self.context.log.warn(\n            f\"[WARNING]: The following targets are not marked with the tag name `{tag_name}`, \"\n            f\"but are dependencies of targets that are type checked. MyPy will check these dependencies, \"\n            f\"inferring `Any` where possible. You are encouraged to properly type check \"\n            f\"these dependencies.\\n{formatted_targets}\"\n        )\n\n    def _calculate_python_sources(self, target_roots: Iterable[Target]) -> List[str]:\n        \"\"\"Filter targets to generate a set of source files from the given targets.\"\"\"\n        all_targets = {\n            tgt\n            for tgt in Target.closure_for_targets(target_roots=target_roots)\n            if self.is_non_synthetic_python_target(tgt)\n        }\n        whitelist_tag_name = self.get_options().whitelist_tag_name\n        if whitelist_tag_name:\n            tagged_targets = {tgt for tgt in all_targets if whitelist_tag_name in tgt.tags}\n            eval_targets = tagged_targets\n            if self.get_options().verbose:\n                self._check_for_untagged_dependencies(\n                    tagged_target_roots={tgt for tgt in tagged_targets if tgt in target_roots},\n                    tag_name=whitelist_tag_name,\n                )\n        else:\n            eval_targets = all_targets\n\n        sources: Set[str] = set()\n        for target in eval_targets:\n            sources.update(\n                source\n                for source in target.sources_relative_to_buildroot()\n                if os.path.splitext(source)[1] == self._PYTHON_SOURCE_EXTENSION\n            )\n        return list(sorted(sources))\n\n    def _collect_source_roots(self):\n        # Collect the set of directories in which there are Python sources (whether part of\n        # the target roots or transitive dependencies.)\n        source_roots = set()\n        for target in self.context.targets(self.is_python_target):\n            if not target.has_sources(self._PYTHON_SOURCE_EXTENSION):\n                continue\n            source_roots.add(target.target_base)\n        return source_roots\n\n    @memoized_property\n    def _interpreter_cache(self):\n        return PythonInterpreterCache.global_instance()\n\n    @memoized_property\n    def _mypy_subsystem(self):\n        return MyPy.global_instance()\n\n    def _get_mypy_pex(self, py3_interpreter: PythonInterpreter, *extra_pexes: PEX) -> PEX:\n        mypy_version = self._mypy_subsystem.options.version\n        extras_hash = hash_utils.hash_all(\n            hash_utils.hash_dir(Path(extra_pex.path())) for extra_pex in extra_pexes\n        )\n\n        path = Path(self.workdir, str(py3_interpreter.identity), f\"{mypy_version}-{extras_hash}\")\n        pex_dir = str(path)\n        if not path.is_dir():\n            mypy_requirement_pex = self.resolve_requirement_strings(py3_interpreter, [mypy_version])\n            pex_info = PexInfo.default()\n            pex_info.entry_point = \"pants_mypy_launcher\"\n            with self.merged_pex(\n                path=pex_dir,\n                pex_info=pex_info,\n                interpreter=py3_interpreter,\n                pexes=[mypy_requirement_pex, *extra_pexes],\n            ) as builder:\n                with temporary_file(binary_mode=False) as exe_fp:\n                    # MyPy searches for types for a package in packages containing a `py.types`\n                    # marker file or else in a sibling `<package>-stubs` package as per PEP-0561.\n                    # Going further than that PEP, MyPy restricts its search to `site-packages`.\n                    # Since PEX deliberately isolates itself from `site-packages` as part of its\n                    # raison d'etre, we monkey-patch `site.getsitepackages` to look inside the\n                    # scrubbed PEX sys.path before handing off to `mypy`.\n                    #\n                    # As a complication, MyPy does its own validation to ensure packages aren't\n                    # both available in site-packages and on the PYTHONPATH. As such, we elide all\n                    # PYTHONPATH entries from artificial site-packages we set up since MyPy will\n                    # manually scan PYTHONPATH outside this PEX to find packages.\n                    #\n                    # See:\n                    #   https://mypy.readthedocs.io/en/stable/installed_packages.html#installed-packages\n                    #   https://www.python.org/dev/peps/pep-0561/#stub-only-packages\n                    exe_fp.write(\n                        dedent(\n                            \"\"\"\n                            import os\n                            import runpy\n                            import site\n                            import sys\n\n                            PYTHONPATH = frozenset(\n                                os.path.realpath(p)\n                                for p in os.environ.get('PYTHONPATH', '').split(os.pathsep)\n                            )\n                            \n                            site.getsitepackages = lambda: [\n                                p for p in sys.path if os.path.realpath(p) not in PYTHONPATH\n                            ]\n\n                            runpy.run_module('mypy', run_name='__main__')\n                            \"\"\"\n                        )\n                    )\n                    exe_fp.flush()\n                    builder.set_executable(\n                        filename=exe_fp.name, env_filename=f\"{pex_info.entry_point}.py\"\n                    )\n                builder.freeze(bytecode_compile=False)\n\n        return PEX(pex_dir, py3_interpreter)\n\n    def execute(self):\n        mypy_interpreter = self.find_mypy_interpreter()\n        if not mypy_interpreter:\n            raise TaskError(\n                f\"Unable to find a Python {self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT} \"\n                f\"interpreter (required for mypy).\"\n            )\n\n        sources = self._calculate_python_sources(self.context.target_roots)\n        if not sources:\n            self.context.log.debug(\"No Python sources to check.\")\n            return\n\n        # Determine interpreter used by the sources so we can tell mypy.\n        interpreter_for_targets = self._interpreter_cache.select_interpreter_for_targets(\n            self.context.target_roots\n        )\n        if not interpreter_for_targets:\n            raise TaskError(\"No Python interpreter compatible with specified sources.\")\n\n        extra_pexes = []\n        if self.get_options().include_requirements:\n            if interpreter_for_targets.identity.matches(\n                self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT\n            ):\n                extra_pexes.append(\n                    self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)\n                )\n                mypy_interpreter = interpreter_for_targets\n            else:\n                self.context.log.warn(\n                    f\"The --include-requirements option is set, but the current target's requirements have \"\n                    f\"been resolved for {interpreter_for_targets.identity} which is not compatible with mypy \"\n                    f\"which needs {self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT}: omitting resolved \"\n                    f\"requirements from the mypy PYTHONPATH.\"\n                )\n\n        with temporary_file_path() as sources_list_path:\n            with open(sources_list_path, \"w\") as f:\n                for source in sources:\n                    f.write(f\"{source}\\n\")\n            # Construct the mypy command line.\n            cmd = [f\"--python-version={interpreter_for_targets.identity.python}\"]\n\n            config = self._mypy_subsystem.options.config\n            if config:\n                cmd.append(f\"--config-file={os.path.join(get_buildroot(), config)}\")\n            cmd.extend(self._mypy_subsystem.options.args)\n            cmd.append(f\"@{sources_list_path}\")\n\n            with self.context.new_workunit(name=\"create_mypy_pex\", labels=[WorkUnitLabel.PREP]):\n                mypy_pex = self._get_mypy_pex(mypy_interpreter, *extra_pexes)\n\n            # Collect source roots for the targets being checked.\n            buildroot = Path(get_buildroot())\n            sources_path = os.pathsep.join(\n                str(buildroot.joinpath(root)) for root in self._collect_source_roots()\n            )\n\n            # Execute mypy.\n            with self.context.new_workunit(\n                name=\"check\",\n                labels=[WorkUnitLabel.TOOL, WorkUnitLabel.RUN],\n                cmd=\" \".join(mypy_pex.cmdline(cmd)),\n            ) as workunit:\n                returncode = mypy_pex.run(\n                    cmd,\n                    env=dict(PYTHONPATH=sources_path, PEX_INHERIT_PATH=\"fallback\"),\n                    stdout=workunit.output(\"stdout\"),\n                    stderr=workunit.output(\"stderr\"),\n                )\n                if returncode != 0:\n                    raise MypyTaskError(f\"mypy failed: code={returncode}\")\n", "path": "contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py"}]}