We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
CLAPACK.js is missing from the dev branch
For some reason, I started to see the CLAPACK.js missing error when using pyodide from https://cdn.jsdelivr.net/pyodide/dev/full/:

```
Couldn't load package from URL https://cdn.jsdelivr.net/pyodide/dev/full/CLAPACK.js
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pyodide_build/buildall.py`
Content:
```
1 #!/usr/bin/env python3
2 
3 """
4 Build all of the packages in a given directory.
5 """
6 
7 import argparse
8 from functools import total_ordering
9 import json
10 from pathlib import Path
11 from queue import Queue, PriorityQueue
12 import shutil
13 import subprocess
14 import sys
15 from threading import Thread
16 from time import sleep
17 from typing import Dict, Set, Optional, List
18 
19 from . import common
20 
21 
22 @total_ordering
23 class Package:
24     def __init__(self, pkgdir: Path):
25         self.pkgdir = pkgdir
26 
27         pkgpath = pkgdir / "meta.yaml"
28         if not pkgpath.is_file():
29             raise ValueError(f"Directory {pkgdir} does not contain meta.yaml")
30 
31         self.meta: dict = common.parse_package(pkgpath)
32         self.name: str = self.meta["package"]["name"]
33         self.library: bool = self.meta.get("build", {}).get("library", False)
34 
35         assert self.name == pkgdir.stem
36 
37         self.dependencies: List[str] = self.meta.get("requirements", {}).get("run", [])
38         self.unbuilt_dependencies: Set[str] = set(self.dependencies)
39         self.dependents: Set[str] = set()
40 
41     def build(self, outputdir: Path, args) -> None:
42         with open(self.pkgdir / "build.log", "w") as f:
43             if self.library:
44                 p = subprocess.run(
45                     ["make"],
46                     cwd=self.pkgdir,
47                     check=False,
48                     stdout=f,
49                     stderr=subprocess.STDOUT,
50                 )
51             else:
52                 p = subprocess.run(
53                     [
54                         sys.executable,
55                         "-m",
56                         "pyodide_build",
57                         "buildpkg",
58                         str(self.pkgdir / "meta.yaml"),
59                         "--package_abi",
60                         str(args.package_abi),
61                         "--cflags",
62                         args.cflags,
63                         "--ldflags",
64                         args.ldflags,
65                         "--target",
66                         args.target,
67                         "--install-dir",
68                         args.install_dir,
69                     ],
70                     check=False,
71                     stdout=f,
72                     stderr=subprocess.STDOUT,
73                 )
74 
75         with open(self.pkgdir / "build.log", "r") as f:
76             shutil.copyfileobj(f, sys.stdout)
77 
78         p.check_returncode()
79 
80         if not self.library:
81             shutil.copyfile(
82                 self.pkgdir / "build" / (self.name + ".data"),
83                 outputdir / (self.name + ".data"),
84             )
85             shutil.copyfile(
86                 self.pkgdir / "build" / (self.name + ".js"),
87                 outputdir / (self.name + ".js"),
88             )
89 
90     # We use this in the priority queue, which pops off the smallest element.
91     # So we want the smallest element to have the largest number of dependents
92     def __lt__(self, other) -> bool:
93         return len(self.dependents) > len(other.dependents)
94 
95     def __eq__(self, other) -> bool:
96         return len(self.dependents) == len(other.dependents)
97 
98 
99 def generate_dependency_graph(
100     packages_dir: Path, package_list: Optional[str]
101 ) -> Dict[str, Package]:
102     """
103     This generates a dependency graph for the packages listed in package_list.
104     A node in the graph is a Package object defined above, which maintains a
105     list of dependencies and also dependents. That is, each node stores both
106     incoming and outgoing edges.
107 
108     The dependencies and dependents are stored via their name, and we have a
109     lookup table pkg_map: Dict[str, Package] to look up the corresponding
110     Package object. The function returns pkg_map, which contains all packages
111     in the graph as its values.
112 
113     Parameters:
114      - packages_dir: directory that contains packages
115      - package_list: set of packages to build. If None, then all packages in
116        packages_dir are compiled.
117 
118     Returns:
119      - pkg_map: dictionary mapping package names to Package objects
120     """
121 
122     pkg_map: Dict[str, Package] = {}
123 
124     packages: Optional[Set[str]] = common._parse_package_subset(package_list)
125     if packages is None:
126         packages = set(
127             str(x) for x in packages_dir.iterdir() if (x / "meta.yaml").is_file()
128         )
129 
130     while packages:
131         pkgname = packages.pop()
132 
133         pkg = Package(packages_dir / pkgname)
134         pkg_map[pkg.name] = pkg
135 
136         for dep in pkg.dependencies:
137             if pkg_map.get(dep) is None:
138                 packages.add(dep)
139 
140     # Compute dependents
141     for pkg in pkg_map.values():
142         for dep in pkg.dependencies:
143             pkg_map[dep].dependents.add(pkg.name)
144 
145     return pkg_map
146 
147 
148 def build_from_graph(pkg_map: Dict[str, Package], outputdir: Path, args) -> None:
149     """
150     This builds packages in pkg_map in parallel, building at most args.n_jobs
151     packages at once.
152 
153     We have a priority queue of packages we are ready to build (build_queue),
154     where a package is ready to build if all its dependencies are built. The
155     priority is based on the number of dependents --- we prefer to build
156     packages with more dependents first.
157 
158     To build packages in parallel, we use a thread pool of args.n_jobs many
159     threads listening to build_queue. When the thread is free, it takes an
160     item off build_queue and builds it. Once the package is built, it sends the
161     package to the built_queue. The main thread listens to the built_queue and
162     checks if any of the dependents are ready to be built. If so, it add the
163     package to the build queue.
164     """
165 
166     # Insert packages into build_queue. We *must* do this after counting
167     # dependents, because the ordering ought not to change after insertion.
168     build_queue: PriorityQueue = PriorityQueue()
169     for pkg in pkg_map.values():
170         if len(pkg.dependencies) == 0:
171             build_queue.put(pkg)
172 
173     built_queue: Queue = Queue()
174 
175     def builder(n):
176         print(f"Starting thread {n}")
177         while True:
178             pkg = build_queue.get()
179             print(f"Thread {n} building {pkg.name}")
180             try:
181                 pkg.build(outputdir, args)
182             except Exception as e:
183                 built_queue.put(e)
184                 return
185 
186             print(f"Thread {n} built {pkg.name}")
187             built_queue.put(pkg)
188             # Release the GIL so new packages get queued
189             sleep(0.01)
190 
191     for n in range(0, args.n_jobs):
192         Thread(target=builder, args=(n + 1,), daemon=True).start()
193 
194     num_built = 0
195     while num_built < len(pkg_map):
196         pkg = built_queue.get()
197         if isinstance(pkg, Exception):
198             raise pkg
199 
200         num_built += 1
201 
202         for _dependent in pkg.dependents:
203             dependent = pkg_map[_dependent]
204             dependent.unbuilt_dependencies.remove(pkg.name)
205             if len(dependent.unbuilt_dependencies) == 0:
206                 build_queue.put(dependent)
207 
208 
209 def build_packages(packages_dir: Path, outputdir: Path, args) -> None:
210     pkg_map = generate_dependency_graph(packages_dir, args.only)
211 
212     build_from_graph(pkg_map, outputdir, args)
213 
214     # Build package.json data. The "test" package is built in a different way,
215     # so we hardcode its existence here.
216     #
217     # This is done last so the Makefile can use it as a completion token.
218     package_data: dict = {
219         "dependencies": {"test": []},
220         "import_name_to_package_name": {},
221     }
222 
223     for name, pkg in pkg_map.items():
224         if pkg.library:
225             continue
226 
227         package_data["dependencies"][name] = pkg.dependencies
228         for imp in pkg.meta.get("test", {}).get("imports", [name]):
229             package_data["import_name_to_package_name"][imp] = name
230 
231     with open(outputdir / "packages.json", "w") as fd:
232         json.dump(package_data, fd)
233 
234 
235 def make_parser(parser):
236     parser.description = (
237         "Build all of the packages in a given directory\n\n"
238         "Unless the --only option is provided"
239     )
240     parser.add_argument(
241         "dir",
242         type=str,
243         nargs=1,
244         help="Input directory containing a tree of package definitions",
245     )
246     parser.add_argument(
247         "output",
248         type=str,
249         nargs=1,
250         help="Output directory in which to put all built packages",
251     )
252     parser.add_argument(
253         "--package_abi",
254         type=int,
255         required=True,
256         help="The ABI number for the packages to be built",
257     )
258     parser.add_argument(
259         "--cflags",
260         type=str,
261         nargs="?",
262         default=common.DEFAULTCFLAGS,
263         help="Extra compiling flags",
264     )
265     parser.add_argument(
266         "--ldflags",
267         type=str,
268         nargs="?",
269         default=common.DEFAULTLDFLAGS,
270         help="Extra linking flags",
271     )
272     parser.add_argument(
273         "--target",
274         type=str,
275         nargs="?",
276         default=common.TARGETPYTHON,
277         help="The path to the target Python installation",
278     )
279     parser.add_argument(
280         "--install-dir",
281         type=str,
282         nargs="?",
283         default="",
284         help=(
285             "Directory for installing built host packages. Defaults to setup.py "
286             "default. Set to 'skip' to skip installation. Installation is "
287             "needed if you want to build other packages that depend on this one."
288         ),
289     )
290     parser.add_argument(
291         "--only",
292         type=str,
293         nargs="?",
294         default=None,
295         help=(
296             "Only build the specified packages, provided as a comma " "separated list"
297         ),
298     )
299     parser.add_argument(
300         "--n-jobs",
301         type=int,
302         nargs="?",
303         default=4,
304         help="Number of packages to build in parallel",
305     )
306     return parser
307 
308 
309 def main(args):
310     packages_dir = Path(args.dir[0]).resolve()
311     outputdir = Path(args.output[0]).resolve()
312     build_packages(packages_dir, outputdir, args)
313 
314 
315 if __name__ == "__main__":
316     parser = make_parser(argparse.ArgumentParser())
317     args = parser.parse_args()
318     main(args)
319
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pyodide_build/buildall.py b/pyodide_build/buildall.py
--- a/pyodide_build/buildall.py
+++ b/pyodide_build/buildall.py
@@ -220,11 +220,15 @@
         "import_name_to_package_name": {},
     }
 
+    libraries = [pkg.name for pkg in pkg_map.values() if pkg.library]
+
     for name, pkg in pkg_map.items():
         if pkg.library:
             continue
 
-        package_data["dependencies"][name] = pkg.dependencies
+        package_data["dependencies"][name] = [
+            x for x in pkg.dependencies if x not in libraries
+        ]
         for imp in pkg.meta.get("test", {}).get("imports", [name]):
             package_data["import_name_to_package_name"][imp] = name

{"golden_diff": "diff --git a/pyodide_build/buildall.py b/pyodide_build/buildall.py\n--- a/pyodide_build/buildall.py\n+++ b/pyodide_build/buildall.py\n@@ -220,11 +220,15 @@\n         \"import_name_to_package_name\": {},\n     }\n \n+    libraries = [pkg.name for pkg in pkg_map.values() if pkg.library]\n+\n     for name, pkg in pkg_map.items():\n         if pkg.library:\n             continue\n \n-        package_data[\"dependencies\"][name] = pkg.dependencies\n+        package_data[\"dependencies\"][name] = [\n+            x for x in pkg.dependencies if x not in libraries\n+        ]\n         for imp in pkg.meta.get(\"test\", {}).get(\"imports\", [name]):\n             package_data[\"import_name_to_package_name\"][imp] = name\n", "issue": "CLAPACK.js is missing from the dev branch\nFor some reason, I started to see the CLAPACK.js missing error when using pyodide from https://cdn.jsdelivr.net/pyodide/dev/full/:\r\n\r\n```\r\nCouldn't load package from URL https://cdn.jsdelivr.net/pyodide/dev/full/CLAPACK.js\r\n```\n", "before_files": [{"content": "#!/usr/bin/env python3\n\n\"\"\"\nBuild all of the packages in a given directory.\n\"\"\"\n\nimport argparse\nfrom functools import total_ordering\nimport json\nfrom pathlib import Path\nfrom queue import Queue, PriorityQueue\nimport shutil\nimport subprocess\nimport sys\nfrom threading import Thread\nfrom time import sleep\nfrom typing import Dict, Set, Optional, List\n\nfrom . import common\n\n\n@total_ordering\nclass Package:\n    def __init__(self, pkgdir: Path):\n        self.pkgdir = pkgdir\n\n        pkgpath = pkgdir / \"meta.yaml\"\n        if not pkgpath.is_file():\n            raise ValueError(f\"Directory {pkgdir} does not contain meta.yaml\")\n\n        self.meta: dict = common.parse_package(pkgpath)\n        self.name: str = self.meta[\"package\"][\"name\"]\n        self.library: bool = self.meta.get(\"build\", {}).get(\"library\", False)\n\n        assert self.name == pkgdir.stem\n\n        self.dependencies: List[str] = self.meta.get(\"requirements\", {}).get(\"run\", [])\n        self.unbuilt_dependencies: Set[str] = set(self.dependencies)\n        self.dependents: Set[str] = set()\n\n    def build(self, outputdir: Path, args) -> None:\n        with open(self.pkgdir / \"build.log\", \"w\") as f:\n            if self.library:\n                p = subprocess.run(\n                    [\"make\"],\n                    cwd=self.pkgdir,\n                    check=False,\n                    stdout=f,\n                    stderr=subprocess.STDOUT,\n                )\n            else:\n                p = subprocess.run(\n                    [\n                        sys.executable,\n                        \"-m\",\n                        \"pyodide_build\",\n                        \"buildpkg\",\n                        str(self.pkgdir / \"meta.yaml\"),\n                        \"--package_abi\",\n                        str(args.package_abi),\n                        \"--cflags\",\n                        args.cflags,\n                        \"--ldflags\",\n                        args.ldflags,\n                        \"--target\",\n                        args.target,\n                        \"--install-dir\",\n                        args.install_dir,\n                    ],\n                    check=False,\n                    stdout=f,\n                    stderr=subprocess.STDOUT,\n                )\n\n        with open(self.pkgdir / \"build.log\", \"r\") as f:\n            shutil.copyfileobj(f, sys.stdout)\n\n        p.check_returncode()\n\n        if not self.library:\n            shutil.copyfile(\n                self.pkgdir / \"build\" / (self.name + \".data\"),\n                outputdir / (self.name + \".data\"),\n            )\n            shutil.copyfile(\n                self.pkgdir / \"build\" / (self.name + \".js\"),\n                outputdir / (self.name + \".js\"),\n            )\n\n    # We use this in the priority queue, which pops off the smallest element.\n    # So we want the smallest element to have the largest number of dependents\n    def __lt__(self, other) -> bool:\n        return len(self.dependents) > len(other.dependents)\n\n    def __eq__(self, other) -> bool:\n        return len(self.dependents) == len(other.dependents)\n\n\ndef generate_dependency_graph(\n    packages_dir: Path, package_list: Optional[str]\n) -> Dict[str, Package]:\n    \"\"\"\n    This generates a dependency graph for the packages listed in package_list.\n    A node in the graph is a Package object defined above, which maintains a\n    list of dependencies and also dependents. That is, each node stores both\n    incoming and outgoing edges.\n\n    The dependencies and dependents are stored via their name, and we have a\n    lookup table pkg_map: Dict[str, Package] to look up the corresponding\n    Package object. The function returns pkg_map, which contains all packages\n    in the graph as its values.\n\n    Parameters:\n     - packages_dir: directory that contains packages\n     - package_list: set of packages to build. If None, then all packages in\n       packages_dir are compiled.\n\n    Returns:\n     - pkg_map: dictionary mapping package names to Package objects\n    \"\"\"\n\n    pkg_map: Dict[str, Package] = {}\n\n    packages: Optional[Set[str]] = common._parse_package_subset(package_list)\n    if packages is None:\n        packages = set(\n            str(x) for x in packages_dir.iterdir() if (x / \"meta.yaml\").is_file()\n        )\n\n    while packages:\n        pkgname = packages.pop()\n\n        pkg = Package(packages_dir / pkgname)\n        pkg_map[pkg.name] = pkg\n\n        for dep in pkg.dependencies:\n            if pkg_map.get(dep) is None:\n                packages.add(dep)\n\n    # Compute dependents\n    for pkg in pkg_map.values():\n        for dep in pkg.dependencies:\n            pkg_map[dep].dependents.add(pkg.name)\n\n    return pkg_map\n\n\ndef build_from_graph(pkg_map: Dict[str, Package], outputdir: Path, args) -> None:\n    \"\"\"\n    This builds packages in pkg_map in parallel, building at most args.n_jobs\n    packages at once.\n\n    We have a priority queue of packages we are ready to build (build_queue),\n    where a package is ready to build if all its dependencies are built. The\n    priority is based on the number of dependents --- we prefer to build\n    packages with more dependents first.\n\n    To build packages in parallel, we use a thread pool of args.n_jobs many\n    threads listening to build_queue. When the thread is free, it takes an\n    item off build_queue and builds it. Once the package is built, it sends the\n    package to the built_queue. The main thread listens to the built_queue and\n    checks if any of the dependents are ready to be built. If so, it add the\n    package to the build queue.\n    \"\"\"\n\n    # Insert packages into build_queue. We *must* do this after counting\n    # dependents, because the ordering ought not to change after insertion.\n    build_queue: PriorityQueue = PriorityQueue()\n    for pkg in pkg_map.values():\n        if len(pkg.dependencies) == 0:\n            build_queue.put(pkg)\n\n    built_queue: Queue = Queue()\n\n    def builder(n):\n        print(f\"Starting thread {n}\")\n        while True:\n            pkg = build_queue.get()\n            print(f\"Thread {n} building {pkg.name}\")\n            try:\n                pkg.build(outputdir, args)\n            except Exception as e:\n                built_queue.put(e)\n                return\n\n            print(f\"Thread {n} built {pkg.name}\")\n            built_queue.put(pkg)\n            # Release the GIL so new packages get queued\n            sleep(0.01)\n\n    for n in range(0, args.n_jobs):\n        Thread(target=builder, args=(n + 1,), daemon=True).start()\n\n    num_built = 0\n    while num_built < len(pkg_map):\n        pkg = built_queue.get()\n        if isinstance(pkg, Exception):\n            raise pkg\n\n        num_built += 1\n\n        for _dependent in pkg.dependents:\n            dependent = pkg_map[_dependent]\n            dependent.unbuilt_dependencies.remove(pkg.name)\n            if len(dependent.unbuilt_dependencies) == 0:\n                build_queue.put(dependent)\n\n\ndef build_packages(packages_dir: Path, outputdir: Path, args) -> None:\n    pkg_map = generate_dependency_graph(packages_dir, args.only)\n\n    build_from_graph(pkg_map, outputdir, args)\n\n    # Build package.json data. The \"test\" package is built in a different way,\n    # so we hardcode its existence here.\n    #\n    # This is done last so the Makefile can use it as a completion token.\n    package_data: dict = {\n        \"dependencies\": {\"test\": []},\n        \"import_name_to_package_name\": {},\n    }\n\n    for name, pkg in pkg_map.items():\n        if pkg.library:\n            continue\n\n        package_data[\"dependencies\"][name] = pkg.dependencies\n        for imp in pkg.meta.get(\"test\", {}).get(\"imports\", [name]):\n            package_data[\"import_name_to_package_name\"][imp] = name\n\n    with open(outputdir / \"packages.json\", \"w\") as fd:\n        json.dump(package_data, fd)\n\n\ndef make_parser(parser):\n    parser.description = (\n        \"Build all of the packages in a given directory\\n\\n\"\n        \"Unless the --only option is provided\"\n    )\n    parser.add_argument(\n        \"dir\",\n        type=str,\n        nargs=1,\n        help=\"Input directory containing a tree of package definitions\",\n    )\n    parser.add_argument(\n        \"output\",\n        type=str,\n        nargs=1,\n        help=\"Output directory in which to put all built packages\",\n    )\n    parser.add_argument(\n        \"--package_abi\",\n        type=int,\n        required=True,\n        help=\"The ABI number for the packages to be built\",\n    )\n    parser.add_argument(\n        \"--cflags\",\n        type=str,\n        nargs=\"?\",\n        default=common.DEFAULTCFLAGS,\n        help=\"Extra compiling flags\",\n    )\n    parser.add_argument(\n        \"--ldflags\",\n        type=str,\n        nargs=\"?\",\n        default=common.DEFAULTLDFLAGS,\n        help=\"Extra linking flags\",\n    )\n    parser.add_argument(\n        \"--target\",\n        type=str,\n        nargs=\"?\",\n        default=common.TARGETPYTHON,\n        help=\"The path to the target Python installation\",\n    )\n    parser.add_argument(\n        \"--install-dir\",\n        type=str,\n        nargs=\"?\",\n        default=\"\",\n        help=(\n            \"Directory for installing built host packages. Defaults to setup.py \"\n            \"default. Set to 'skip' to skip installation. Installation is \"\n            \"needed if you want to build other packages that depend on this one.\"\n        ),\n    )\n    parser.add_argument(\n        \"--only\",\n        type=str,\n        nargs=\"?\",\n        default=None,\n        help=(\n            \"Only build the specified packages, provided as a comma \" \"separated list\"\n        ),\n    )\n    parser.add_argument(\n        \"--n-jobs\",\n        type=int,\n        nargs=\"?\",\n        default=4,\n        help=\"Number of packages to build in parallel\",\n    )\n    return parser\n\n\ndef main(args):\n    packages_dir = Path(args.dir[0]).resolve()\n    outputdir = Path(args.output[0]).resolve()\n    build_packages(packages_dir, outputdir, args)\n\n\nif __name__ == \"__main__\":\n    parser = make_parser(argparse.ArgumentParser())\n    args = parser.parse_args()\n    main(args)\n", "path": "pyodide_build/buildall.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\n\"\"\"\nBuild all of the packages in a given directory.\n\"\"\"\n\nimport argparse\nfrom functools import total_ordering\nimport json\nfrom pathlib import Path\nfrom queue import Queue, PriorityQueue\nimport shutil\nimport subprocess\nimport sys\nfrom threading import Thread\nfrom time import sleep\nfrom typing import Dict, Set, Optional, List\n\nfrom . import common\n\n\n@total_ordering\nclass Package:\n    def __init__(self, pkgdir: Path):\n        self.pkgdir = pkgdir\n\n        pkgpath = pkgdir / \"meta.yaml\"\n        if not pkgpath.is_file():\n            raise ValueError(f\"Directory {pkgdir} does not contain meta.yaml\")\n\n        self.meta: dict = common.parse_package(pkgpath)\n        self.name: str = self.meta[\"package\"][\"name\"]\n        self.library: bool = self.meta.get(\"build\", {}).get(\"library\", False)\n\n        assert self.name == pkgdir.stem\n\n        self.dependencies: List[str] = self.meta.get(\"requirements\", {}).get(\"run\", [])\n        self.unbuilt_dependencies: Set[str] = set(self.dependencies)\n        self.dependents: Set[str] = set()\n\n    def build(self, outputdir: Path, args) -> None:\n        with open(self.pkgdir / \"build.log\", \"w\") as f:\n            if self.library:\n                p = subprocess.run(\n                    [\"make\"],\n                    cwd=self.pkgdir,\n                    check=False,\n                    stdout=f,\n                    stderr=subprocess.STDOUT,\n                )\n            else:\n                p = subprocess.run(\n                    [\n                        sys.executable,\n                        \"-m\",\n                        \"pyodide_build\",\n                        \"buildpkg\",\n                        str(self.pkgdir / \"meta.yaml\"),\n                        \"--package_abi\",\n                        str(args.package_abi),\n                        \"--cflags\",\n                        args.cflags,\n                        \"--ldflags\",\n                        args.ldflags,\n                        \"--target\",\n                        args.target,\n                        \"--install-dir\",\n                        args.install_dir,\n                    ],\n                    check=False,\n                    stdout=f,\n                    stderr=subprocess.STDOUT,\n                )\n\n        with open(self.pkgdir / \"build.log\", \"r\") as f:\n            shutil.copyfileobj(f, sys.stdout)\n\n        p.check_returncode()\n\n        if not self.library:\n            shutil.copyfile(\n                self.pkgdir / \"build\" / (self.name + \".data\"),\n                outputdir / (self.name + \".data\"),\n            )\n            shutil.copyfile(\n                self.pkgdir / \"build\" / (self.name + \".js\"),\n                outputdir / (self.name + \".js\"),\n            )\n\n    # We use this in the priority queue, which pops off the smallest element.\n    # So we want the smallest element to have the largest number of dependents\n    def __lt__(self, other) -> bool:\n        return len(self.dependents) > len(other.dependents)\n\n    def __eq__(self, other) -> bool:\n        return len(self.dependents) == len(other.dependents)\n\n\ndef generate_dependency_graph(\n    packages_dir: Path, package_list: Optional[str]\n) -> Dict[str, Package]:\n    \"\"\"\n    This generates a dependency graph for the packages listed in package_list.\n    A node in the graph is a Package object defined above, which maintains a\n    list of dependencies and also dependents. That is, each node stores both\n    incoming and outgoing edges.\n\n    The dependencies and dependents are stored via their name, and we have a\n    lookup table pkg_map: Dict[str, Package] to look up the corresponding\n    Package object. The function returns pkg_map, which contains all packages\n    in the graph as its values.\n\n    Parameters:\n     - packages_dir: directory that contains packages\n     - package_list: set of packages to build. If None, then all packages in\n       packages_dir are compiled.\n\n    Returns:\n     - pkg_map: dictionary mapping package names to Package objects\n    \"\"\"\n\n    pkg_map: Dict[str, Package] = {}\n\n    packages: Optional[Set[str]] = common._parse_package_subset(package_list)\n    if packages is None:\n        packages = set(\n            str(x) for x in packages_dir.iterdir() if (x / \"meta.yaml\").is_file()\n        )\n\n    while packages:\n        pkgname = packages.pop()\n\n        pkg = Package(packages_dir / pkgname)\n        pkg_map[pkg.name] = pkg\n\n        for dep in pkg.dependencies:\n            if pkg_map.get(dep) is None:\n                packages.add(dep)\n\n    # Compute dependents\n    for pkg in pkg_map.values():\n        for dep in pkg.dependencies:\n            pkg_map[dep].dependents.add(pkg.name)\n\n    return pkg_map\n\n\ndef build_from_graph(pkg_map: Dict[str, Package], outputdir: Path, args) -> None:\n    \"\"\"\n    This builds packages in pkg_map in parallel, building at most args.n_jobs\n    packages at once.\n\n    We have a priority queue of packages we are ready to build (build_queue),\n    where a package is ready to build if all its dependencies are built. The\n    priority is based on the number of dependents --- we prefer to build\n    packages with more dependents first.\n\n    To build packages in parallel, we use a thread pool of args.n_jobs many\n    threads listening to build_queue. When the thread is free, it takes an\n    item off build_queue and builds it. Once the package is built, it sends the\n    package to the built_queue. The main thread listens to the built_queue and\n    checks if any of the dependents are ready to be built. If so, it add the\n    package to the build queue.\n    \"\"\"\n\n    # Insert packages into build_queue. We *must* do this after counting\n    # dependents, because the ordering ought not to change after insertion.\n    build_queue: PriorityQueue = PriorityQueue()\n    for pkg in pkg_map.values():\n        if len(pkg.dependencies) == 0:\n            build_queue.put(pkg)\n\n    built_queue: Queue = Queue()\n\n    def builder(n):\n        print(f\"Starting thread {n}\")\n        while True:\n            pkg = build_queue.get()\n            print(f\"Thread {n} building {pkg.name}\")\n            try:\n                pkg.build(outputdir, args)\n            except Exception as e:\n                built_queue.put(e)\n                return\n\n            print(f\"Thread {n} built {pkg.name}\")\n            built_queue.put(pkg)\n            # Release the GIL so new packages get queued\n            sleep(0.01)\n\n    for n in range(0, args.n_jobs):\n        Thread(target=builder, args=(n + 1,), daemon=True).start()\n\n    num_built = 0\n    while num_built < len(pkg_map):\n        pkg = built_queue.get()\n        if isinstance(pkg, Exception):\n            raise pkg\n\n        num_built += 1\n\n        for _dependent in pkg.dependents:\n            dependent = pkg_map[_dependent]\n            dependent.unbuilt_dependencies.remove(pkg.name)\n            if len(dependent.unbuilt_dependencies) == 0:\n                build_queue.put(dependent)\n\n\ndef build_packages(packages_dir: Path, outputdir: Path, args) -> None:\n    pkg_map = generate_dependency_graph(packages_dir, args.only)\n\n    build_from_graph(pkg_map, outputdir, args)\n\n    # Build package.json data. The \"test\" package is built in a different way,\n    # so we hardcode its existence here.\n    #\n    # This is done last so the Makefile can use it as a completion token.\n    package_data: dict = {\n        \"dependencies\": {\"test\": []},\n        \"import_name_to_package_name\": {},\n    }\n\n    libraries = [pkg.name for pkg in pkg_map.values() if pkg.library]\n\n    for name, pkg in pkg_map.items():\n        if pkg.library:\n            continue\n\n        package_data[\"dependencies\"][name] = [\n            x for x in pkg.dependencies if x not in libraries\n        ]\n        for imp in pkg.meta.get(\"test\", {}).get(\"imports\", [name]):\n            package_data[\"import_name_to_package_name\"][imp] = name\n\n    with open(outputdir / \"packages.json\", \"w\") as fd:\n        json.dump(package_data, fd)\n\n\ndef make_parser(parser):\n    parser.description = (\n        \"Build all of the packages in a given directory\\n\\n\"\n        \"Unless the --only option is provided\"\n    )\n    parser.add_argument(\n        \"dir\",\n        type=str,\n        nargs=1,\n        help=\"Input directory containing a tree of package definitions\",\n    )\n    parser.add_argument(\n        \"output\",\n        type=str,\n        nargs=1,\n        help=\"Output directory in which to put all built packages\",\n    )\n    parser.add_argument(\n        \"--package_abi\",\n        type=int,\n        required=True,\n        help=\"The ABI number for the packages to be built\",\n    )\n    parser.add_argument(\n        \"--cflags\",\n        type=str,\n        nargs=\"?\",\n        default=common.DEFAULTCFLAGS,\n        help=\"Extra compiling flags\",\n    )\n    parser.add_argument(\n        \"--ldflags\",\n        type=str,\n        nargs=\"?\",\n        default=common.DEFAULTLDFLAGS,\n        help=\"Extra linking flags\",\n    )\n    parser.add_argument(\n        \"--target\",\n        type=str,\n        nargs=\"?\",\n        default=common.TARGETPYTHON,\n        help=\"The path to the target Python installation\",\n    )\n    parser.add_argument(\n        \"--install-dir\",\n        type=str,\n        nargs=\"?\",\n        default=\"\",\n        help=(\n            \"Directory for installing built host packages. Defaults to setup.py \"\n            \"default. Set to 'skip' to skip installation. Installation is \"\n            \"needed if you want to build other packages that depend on this one.\"\n        ),\n    )\n    parser.add_argument(\n        \"--only\",\n        type=str,\n        nargs=\"?\",\n        default=None,\n        help=(\n            \"Only build the specified packages, provided as a comma \" \"separated list\"\n        ),\n    )\n    parser.add_argument(\n        \"--n-jobs\",\n        type=int,\n        nargs=\"?\",\n        default=4,\n        help=\"Number of packages to build in parallel\",\n    )\n    return parser\n\n\ndef main(args):\n    packages_dir = Path(args.dir[0]).resolve()\n    outputdir = Path(args.output[0]).resolve()\n    build_packages(packages_dir, outputdir, args)\n\n\nif __name__ == \"__main__\":\n    parser = make_parser(argparse.ArgumentParser())\n    args = parser.parse_args()\n    main(args)\n", "path": "pyodide_build/buildall.py"}]}