We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
ModuleNotFound error with latest release
**Environment**
<!-- Please copy paste the output of running `composer_collect_env` below-->

mosaicml==0.13.0

Running `composer_collect_env` fails with:

```
❯ composer_collect_env
Traceback (most recent call last):
  File "/Users/evanw/.virtualenvs/llm/bin/composer_collect_env", line 5, in <module>
    from composer.utils.collect_env import main
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/__init__.py", line 10, in <module>
    from composer.trainer import Trainer
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/trainer/__init__.py", line 6, in <module>
    from composer.trainer.trainer import Trainer
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/trainer/trainer.py", line 33, in <module>
    from composer.callbacks import CheckpointSaver, OptimizerMonitor
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/callbacks/__init__.py", line 12, in <module>
    from composer.callbacks.health_checker import HealthChecker
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/callbacks/health_checker.py", line 20, in <module>
    from slack_sdk.webhook import WebhookClient
ModuleNotFoundError: No module named 'slack_sdk'
```

<!-- if your bug is CUDA related, please also provide your CUDA version -->

**To reproduce**

Steps to reproduce the behavior:

1. Install `mosaicml==0.13.0`
2. Run `python -c 'from composer.metrics.nlp import Perplexity'` (or something similar)

You'll get the same stacktrace as above:

```
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/__init__.py", line 10, in <module>
    from composer.trainer import Trainer
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/trainer/__init__.py", line 6, in <module>
    from composer.trainer.trainer import Trainer
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/trainer/trainer.py", line 33, in <module>
    from composer.callbacks import CheckpointSaver, OptimizerMonitor
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/callbacks/__init__.py", line 12, in <module>
    from composer.callbacks.health_checker import HealthChecker
  File "/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/callbacks/health_checker.py", line 20, in <module>
    from slack_sdk.webhook import WebhookClient
ModuleNotFoundError: No module named 'slack_sdk'
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `composer/callbacks/health_checker.py`
Content:
```
1 # Copyright 2022 MosaicML Composer authors
2 # SPDX-License-Identifier: Apache-2.0
3 
4 """Check GPU Health during training."""
5 import logging
6 from collections import deque
7 from datetime import datetime
8 from typing import List, Optional, Tuple
9 
10 import torch
11 
12 try:
13     import pynvml
14 except ImportError:
15     pynvml = None
16 
17 import os
18 
19 import numpy as np
20 from slack_sdk.webhook import WebhookClient
21 
22 from composer.core import Callback, State
23 from composer.core.time import Timestamp
24 from composer.loggers import Logger
25 from composer.utils import dist
26 
27 log = logging.getLogger(__name__)
28 
29 __all__ = ['HealthChecker']
30 
31 
32 class HealthChecker(Callback):
33     """Checks for GPU health.
34 
35     This callback checks for GPU health by tracking and alerting for abnormal
36     GPU utilizations.
37 
38     For example, if the average utilization during the observation window is,
39     [30, 30, 45], then the range (45-30=15) would exceed a threshold of 10%.
40 
41     Args:
42         threshold (float, optional): Threshold of GPU utilization range to
43             trigger an alert. Defaults to 10.
44         sample_freq (int, optional): Sample frequency in seconds. Default: 5.
45         window_size (int, optional): Window size in seconds. HealthChecker will
46             check for abnormalities at this frequency. Default: 120.
47         wait (int, optional): Seconds to wait for starting to sample. Default: 120.
48         slack_webhook_url (str, optional): Slack URL to send alerts. Can also
49             be set with the SLACK_WEBHOOK_URL environment variable. Default: None
50         test_mode (bool, optional): If True, will send a test alert at the first check.
51             Default: False
52     """
53 
54     def __init__(
55         self,
56         threshold: float = 10,
57         sample_freq: int = 5,
58         window_size: int = 120,
59         wait: int = 120,
60         slack_webhook_url: Optional[str] = None,
61         test_mode: bool = False,
62     ) -> None:
63         self.sample_freq = sample_freq
64         self.window_size = window_size
65         self.wait = wait
66         self.slack_webhook_url = slack_webhook_url
67         self.test_mode = test_mode
68 
69         if not self.slack_webhook_url:
70             self.slack_webhook_url = os.environ.get('SLACK_WEBHOOK_URL', None)
71 
72         self.last_sample = 0
73         self.last_check = 0
74 
75         self.metrics = []
76         if self._is_available():
77             self.metrics.append(GPUUtilization(threshold))
78 
79     def init(self, state: State, logger: Logger) -> None:
80         pass
81 
82     def after_train_batch(self, state: State, logger: Logger):
83         if not self.metrics:
84             return
85 
86         if self._sample(state.timestamp):
87             for metric in self.metrics:
88                 metric.sample()
89 
90         if self._check(state.timestamp):
91             for metric in self.metrics:
92                 message, alert = metric.check()
93                 if self.test_mode and message:
94                     alert = True
95                     message = '[**THIS IS A TEST**]' + message
96                 if alert and not metric.alerted:
97                     self._alert(message, state)
98                     metric.alerted = True
99                 metric.clear()
100 
101     def _sample(self, timestamp: Timestamp) -> bool:
102         now = timestamp.total_wct.seconds
103 
104         if now < self.wait:
105             return False
106 
107         if now - self.last_sample >= self.sample_freq:
108             self.last_sample = now
109             return True
110 
111         return False
112 
113     def _check(self, timestamp: Timestamp) -> bool:
114         now = timestamp.total_wct.seconds
115 
116         if now - self.last_check >= self.window_size:
117             self.last_check = now
118             return True
119         return False
120 
121     def _alert(self, message: str, state: State) -> None:
122         prefix = '[{now}][{run_name}][node_rank={node_rank}]'.format(
123             now=datetime.now(),
124             run_name=state.run_name,
125             node_rank=dist.get_node_rank(),
126         )
127 
128         node_name = os.environ.get('NODENAME', None)
129         if node_name is not None:
130             prefix += f'[node={node_name}]'
131 
132         message = prefix + ' : ' + message
133 
134         logging.warning(message)
135         if self.slack_webhook_url:
136             client = WebhookClient(url=self.slack_webhook_url)
137             client.send(text=message)
138 
139     @staticmethod
140     def _is_available() -> bool:
141         if not torch.cuda.is_available():
142             return False
143         try:
144             pynvml.nvmlInit()  # type: ignore
145             return True
146         except pynvml.NVMLError_LibraryNotFound:  # type: ignore
147             logging.warning('NVML not found, disabling GPU health checking')
148         except ImportError:
149             logging.warning('pynvml library not found, disabling GPU health checking.')
150         except Exception as e:
151             logging.warning(f'Error initializing NVML: {e}')
152 
153         return False
154 
155 
156 class GPUUtilization:
157     """GPU Utilization Metric."""
158 
159     def __init__(self, threshold=10) -> None:
160         self.samples = deque()
161         self.threshold = threshold
162         self.alerted = False
163 
164     def sample(self) -> None:
165         if dist.get_local_rank() == 0:
166             sample = self._sample()
167             if sample is not None:
168                 self.samples.append(sample)
169 
170     def _sample(self) -> Optional[List]:
171         try:
172             samples = []
173             device_count = pynvml.nvmlDeviceGetCount()  # type: ignore
174             for i in range(device_count):
175                 handle = pynvml.nvmlDeviceGetHandleByIndex(i)  # type: ignore
176                 samples.append(pynvml.nvmlDeviceGetUtilizationRates(handle).gpu)  # type: ignore
177         except pynvml.NVMLError:  # type: ignore
178             return None
179         return samples
180 
181     def check(self) -> Tuple[Optional[str], bool]:
182         if dist.get_local_rank() == 0:
183             average_sample = np.nanmean(list(self.samples), axis=0)
184             if np.nanmax(average_sample) - np.nanmin(average_sample) > self.threshold:
185                 message = f'Abnormal GPU utilizations: {average_sample}'
186                 return message, True
187             else:
188                 message = f':+1: Normal GPU utilizations: {average_sample}'
189                 return message, False
190         return None, False
191 
192     def clear(self) -> None:
193         self.samples.clear()
194
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/composer/callbacks/health_checker.py b/composer/callbacks/health_checker.py
--- a/composer/callbacks/health_checker.py
+++ b/composer/callbacks/health_checker.py
@@ -3,26 +3,18 @@
 
 """Check GPU Health during training."""
 import logging
+import os
 from collections import deque
 from datetime import datetime
 from typing import List, Optional, Tuple
 
-import torch
-
-try:
-    import pynvml
-except ImportError:
-    pynvml = None
-
-import os
-
 import numpy as np
-from slack_sdk.webhook import WebhookClient
+import torch
 
 from composer.core import Callback, State
 from composer.core.time import Timestamp
 from composer.loggers import Logger
-from composer.utils import dist
+from composer.utils import MissingConditionalImportError, dist
 
 log = logging.getLogger(__name__)
 
@@ -69,6 +61,14 @@
         if not self.slack_webhook_url:
             self.slack_webhook_url = os.environ.get('SLACK_WEBHOOK_URL', None)
 
+        if self.slack_webhook_url:
+            # fail fast if missing import
+            try:
+                import slack_sdk
+                del slack_sdk
+            except ImportError as e:
+                raise MissingConditionalImportError('health_checker', 'slack_sdk', None) from e
+
         self.last_sample = 0
         self.last_check = 0
 
@@ -133,6 +133,7 @@
 
         logging.warning(message)
         if self.slack_webhook_url:
+            from slack_sdk.webhook import WebhookClient
             client = WebhookClient(url=self.slack_webhook_url)
             client.send(text=message)
 
@@ -141,12 +142,13 @@
         if not torch.cuda.is_available():
             return False
         try:
+            import pynvml
             pynvml.nvmlInit()  # type: ignore
             return True
+        except ImportError:
+            raise MissingConditionalImportError('health_checker', 'pynvml', None)
         except pynvml.NVMLError_LibraryNotFound:  # type: ignore
             logging.warning('NVML not found, disabling GPU health checking')
-        except ImportError:
-            logging.warning('pynvml library not found, disabling GPU health checking.')
         except Exception as e:
             logging.warning(f'Error initializing NVML: {e}')
 
@@ -168,13 +170,18 @@
                 self.samples.append(sample)
 
     def _sample(self) -> Optional[List]:
+        try:
+            import pynvml
+        except ImportError:
+            raise MissingConditionalImportError('health_checker', 'pynvml', None)
+
         try:
             samples = []
-            device_count = pynvml.nvmlDeviceGetCount()  # type: ignore
+            device_count = pynvml.nvmlDeviceGetCount()
             for i in range(device_count):
-                handle = pynvml.nvmlDeviceGetHandleByIndex(i)  # type: ignore
-                samples.append(pynvml.nvmlDeviceGetUtilizationRates(handle).gpu)  # type: ignore
-        except pynvml.NVMLError:  # type: ignore
+                handle = pynvml.nvmlDeviceGetHandleByIndex(i)
+                samples.append(pynvml.nvmlDeviceGetUtilizationRates(handle).gpu)
+        except pynvml.NVMLError:
             return None
         return samples

{"golden_diff": "diff --git a/composer/callbacks/health_checker.py b/composer/callbacks/health_checker.py\n--- a/composer/callbacks/health_checker.py\n+++ b/composer/callbacks/health_checker.py\n@@ -3,26 +3,18 @@\n \n \"\"\"Check GPU Health during training.\"\"\"\n import logging\n+import os\n from collections import deque\n from datetime import datetime\n from typing import List, Optional, Tuple\n \n-import torch\n-\n-try:\n-    import pynvml\n-except ImportError:\n-    pynvml = None\n-\n-import os\n-\n import numpy as np\n-from slack_sdk.webhook import WebhookClient\n+import torch\n \n from composer.core import Callback, State\n from composer.core.time import Timestamp\n from composer.loggers import Logger\n-from composer.utils import dist\n+from composer.utils import MissingConditionalImportError, dist\n \n log = logging.getLogger(__name__)\n \n@@ -69,6 +61,14 @@\n         if not self.slack_webhook_url:\n             self.slack_webhook_url = os.environ.get('SLACK_WEBHOOK_URL', None)\n \n+        if self.slack_webhook_url:\n+            # fail fast if missing import\n+            try:\n+                import slack_sdk\n+                del slack_sdk\n+            except ImportError as e:\n+                raise MissingConditionalImportError('health_checker', 'slack_sdk', None) from e\n+\n         self.last_sample = 0\n         self.last_check = 0\n \n@@ -133,6 +133,7 @@\n \n         logging.warning(message)\n         if self.slack_webhook_url:\n+            from slack_sdk.webhook import WebhookClient\n             client = WebhookClient(url=self.slack_webhook_url)\n             client.send(text=message)\n \n@@ -141,12 +142,13 @@\n         if not torch.cuda.is_available():\n             return False\n         try:\n+            import pynvml\n             pynvml.nvmlInit()  # type: ignore\n             return True\n+        except ImportError:\n+            raise MissingConditionalImportError('health_checker', 'pynvml', None)\n         except pynvml.NVMLError_LibraryNotFound:  # type: ignore\n             logging.warning('NVML not found, disabling GPU health checking')\n-        except ImportError:\n-            logging.warning('pynvml library not found, disabling GPU health checking.')\n         except Exception as e:\n             logging.warning(f'Error initializing NVML: {e}')\n \n@@ -168,13 +170,18 @@\n                 self.samples.append(sample)\n \n     def _sample(self) -> Optional[List]:\n+        try:\n+            import pynvml\n+        except ImportError:\n+            raise MissingConditionalImportError('health_checker', 'pynvml', None)\n+\n         try:\n             samples = []\n-            device_count = pynvml.nvmlDeviceGetCount()  # type: ignore\n+            device_count = pynvml.nvmlDeviceGetCount()\n             for i in range(device_count):\n-                handle = pynvml.nvmlDeviceGetHandleByIndex(i)  # type: ignore\n-                samples.append(pynvml.nvmlDeviceGetUtilizationRates(handle).gpu)  # type: ignore\n-        except pynvml.NVMLError:  # type: ignore\n+                handle = pynvml.nvmlDeviceGetHandleByIndex(i)\n+                samples.append(pynvml.nvmlDeviceGetUtilizationRates(handle).gpu)\n+        except pynvml.NVMLError:\n             return None\n         return samples\n", "issue": "ModuleNotFound error with latest release\n**Environment**\r\n<!-- Please copy paste the output of running `composer_collect_env` below-->\r\n\r\nmosaicml==0.13.0\r\n\r\nRunning `composer_collect_env` fails with:\r\n\r\n```\r\n\u276f composer_collect_env\r\nTraceback (most recent call last):\r\n  File \"/Users/evanw/.virtualenvs/llm/bin/composer_collect_env\", line 5, in <module>\r\n    from composer.utils.collect_env import main\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/__init__.py\", line 10, in <module>\r\n    from composer.trainer import Trainer\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/trainer/__init__.py\", line 6, in <module>\r\n    from composer.trainer.trainer import Trainer\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/trainer/trainer.py\", line 33, in <module>\r\n    from composer.callbacks import CheckpointSaver, OptimizerMonitor\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/callbacks/__init__.py\", line 12, in <module>\r\n    from composer.callbacks.health_checker import HealthChecker\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/callbacks/health_checker.py\", line 20, in <module>\r\n    from slack_sdk.webhook import WebhookClient\r\nModuleNotFoundError: No module named 'slack_sdk'\r\n```\r\n\r\n<!-- if your bug is CUDA related, please also provide your CUDA version -->\r\n\r\n**To reproduce**\r\n\r\nSteps to reproduce the behavior:\r\n\r\n1. Install `mosaicml==0.13.0`\r\n2. Run `python -c 'from composer.metrics.nlp import Perplexity'` (or something similar)\r\n\r\nYou'll get the same stacktrace as above:\r\n\r\n```\r\nTraceback (most recent call last):\r\n  File \"<string>\", line 1, in <module>\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/__init__.py\", line 10, in <module>\r\n    from composer.trainer import Trainer\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/trainer/__init__.py\", line 6, in <module>\r\n    from composer.trainer.trainer import Trainer\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/trainer/trainer.py\", line 33, in <module>\r\n    from composer.callbacks import CheckpointSaver, OptimizerMonitor\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/callbacks/__init__.py\", line 12, in <module>\r\n    from composer.callbacks.health_checker import HealthChecker\r\n  File \"/Users/evanw/.virtualenvs/llm/lib/python3.10/site-packages/composer/callbacks/health_checker.py\", line 20, in <module>\r\n    from slack_sdk.webhook import WebhookClient\r\nModuleNotFoundError: No module named 'slack_sdk'\r\n```\n", "before_files": [{"content": "# Copyright 2022 MosaicML Composer authors\n# SPDX-License-Identifier: Apache-2.0\n\n\"\"\"Check GPU Health during training.\"\"\"\nimport logging\nfrom collections import deque\nfrom datetime import datetime\nfrom typing import List, Optional, Tuple\n\nimport torch\n\ntry:\n    import pynvml\nexcept ImportError:\n    pynvml = None\n\nimport os\n\nimport numpy as np\nfrom slack_sdk.webhook import WebhookClient\n\nfrom composer.core import Callback, State\nfrom composer.core.time import Timestamp\nfrom composer.loggers import Logger\nfrom composer.utils import dist\n\nlog = logging.getLogger(__name__)\n\n__all__ = ['HealthChecker']\n\n\nclass HealthChecker(Callback):\n    \"\"\"Checks for GPU health.\n\n    This callback checks for GPU health by tracking and alerting for abnormal\n    GPU utilizations.\n\n    For example, if the average utilization during the observation window is,\n    [30, 30, 45], then the range (45-30=15) would exceed a threshold of 10%.\n\n    Args:\n        threshold (float, optional): Threshold of GPU utilization range to\n            trigger an alert. Defaults to 10.\n        sample_freq (int, optional): Sample frequency in seconds. Default: 5.\n        window_size (int, optional): Window size in seconds. HealthChecker will\n            check for abnormalities at this frequency. Default: 120.\n        wait (int, optional): Seconds to wait for starting to sample. Default: 120.\n        slack_webhook_url (str, optional): Slack URL to send alerts. Can also\n            be set with the SLACK_WEBHOOK_URL environment variable. Default: None\n        test_mode (bool, optional): If True, will send a test alert at the first check.\n            Default: False\n    \"\"\"\n\n    def __init__(\n        self,\n        threshold: float = 10,\n        sample_freq: int = 5,\n        window_size: int = 120,\n        wait: int = 120,\n        slack_webhook_url: Optional[str] = None,\n        test_mode: bool = False,\n    ) -> None:\n        self.sample_freq = sample_freq\n        self.window_size = window_size\n        self.wait = wait\n        self.slack_webhook_url = slack_webhook_url\n        self.test_mode = test_mode\n\n        if not self.slack_webhook_url:\n            self.slack_webhook_url = os.environ.get('SLACK_WEBHOOK_URL', None)\n\n        self.last_sample = 0\n        self.last_check = 0\n\n        self.metrics = []\n        if self._is_available():\n            self.metrics.append(GPUUtilization(threshold))\n\n    def init(self, state: State, logger: Logger) -> None:\n        pass\n\n    def after_train_batch(self, state: State, logger: Logger):\n        if not self.metrics:\n            return\n\n        if self._sample(state.timestamp):\n            for metric in self.metrics:\n                metric.sample()\n\n        if self._check(state.timestamp):\n            for metric in self.metrics:\n                message, alert = metric.check()\n                if self.test_mode and message:\n                    alert = True\n                    message = '[**THIS IS A TEST**]' + message\n                if alert and not metric.alerted:\n                    self._alert(message, state)\n                    metric.alerted = True\n                metric.clear()\n\n    def _sample(self, timestamp: Timestamp) -> bool:\n        now = timestamp.total_wct.seconds\n\n        if now < self.wait:\n            return False\n\n        if now - self.last_sample >= self.sample_freq:\n            self.last_sample = now\n            return True\n\n        return False\n\n    def _check(self, timestamp: Timestamp) -> bool:\n        now = timestamp.total_wct.seconds\n\n        if now - self.last_check >= self.window_size:\n            self.last_check = now\n            return True\n        return False\n\n    def _alert(self, message: str, state: State) -> None:\n        prefix = '[{now}][{run_name}][node_rank={node_rank}]'.format(\n            now=datetime.now(),\n            run_name=state.run_name,\n            node_rank=dist.get_node_rank(),\n        )\n\n        node_name = os.environ.get('NODENAME', None)\n        if node_name is not None:\n            prefix += f'[node={node_name}]'\n\n        message = prefix + ' : ' + message\n\n        logging.warning(message)\n        if self.slack_webhook_url:\n            client = WebhookClient(url=self.slack_webhook_url)\n            client.send(text=message)\n\n    @staticmethod\n    def _is_available() -> bool:\n        if not torch.cuda.is_available():\n            return False\n        try:\n            pynvml.nvmlInit()  # type: ignore\n            return True\n        except pynvml.NVMLError_LibraryNotFound:  # type: ignore\n            logging.warning('NVML not found, disabling GPU health checking')\n        except ImportError:\n            logging.warning('pynvml library not found, disabling GPU health checking.')\n        except Exception as e:\n            logging.warning(f'Error initializing NVML: {e}')\n\n        return False\n\n\nclass GPUUtilization:\n    \"\"\"GPU Utilization Metric.\"\"\"\n\n    def __init__(self, threshold=10) -> None:\n        self.samples = deque()\n        self.threshold = threshold\n        self.alerted = False\n\n    def sample(self) -> None:\n        if dist.get_local_rank() == 0:\n            sample = self._sample()\n            if sample is not None:\n                self.samples.append(sample)\n\n    def _sample(self) -> Optional[List]:\n        try:\n            samples = []\n            device_count = pynvml.nvmlDeviceGetCount()  # type: ignore\n            for i in range(device_count):\n                handle = pynvml.nvmlDeviceGetHandleByIndex(i)  # type: ignore\n                samples.append(pynvml.nvmlDeviceGetUtilizationRates(handle).gpu)  # type: ignore\n        except pynvml.NVMLError:  # type: ignore\n            return None\n        return samples\n\n    def check(self) -> Tuple[Optional[str], bool]:\n        if dist.get_local_rank() == 0:\n            average_sample = np.nanmean(list(self.samples), axis=0)\n            if np.nanmax(average_sample) - np.nanmin(average_sample) > self.threshold:\n                message = f'Abnormal GPU utilizations: {average_sample}'\n                return message, True\n            else:\n                message = f':+1: Normal GPU utilizations: {average_sample}'\n                return message, False\n        return None, False\n\n    def clear(self) -> None:\n        self.samples.clear()\n", "path": "composer/callbacks/health_checker.py"}], "after_files": [{"content": "# Copyright 2022 MosaicML Composer authors\n# SPDX-License-Identifier: Apache-2.0\n\n\"\"\"Check GPU Health during training.\"\"\"\nimport logging\nimport os\nfrom collections import deque\nfrom datetime import datetime\nfrom typing import List, Optional, Tuple\n\nimport numpy as np\nimport torch\n\nfrom composer.core import Callback, State\nfrom composer.core.time import Timestamp\nfrom composer.loggers import Logger\nfrom composer.utils import MissingConditionalImportError, dist\n\nlog = logging.getLogger(__name__)\n\n__all__ = ['HealthChecker']\n\n\nclass HealthChecker(Callback):\n    \"\"\"Checks for GPU health.\n\n    This callback checks for GPU health by tracking and alerting for abnormal\n    GPU utilizations.\n\n    For example, if the average utilization during the observation window is,\n    [30, 30, 45], then the range (45-30=15) would exceed a threshold of 10%.\n\n    Args:\n        threshold (float, optional): Threshold of GPU utilization range to\n            trigger an alert. Defaults to 10.\n        sample_freq (int, optional): Sample frequency in seconds. Default: 5.\n        window_size (int, optional): Window size in seconds. HealthChecker will\n            check for abnormalities at this frequency. Default: 120.\n        wait (int, optional): Seconds to wait for starting to sample. Default: 120.\n        slack_webhook_url (str, optional): Slack URL to send alerts. Can also\n            be set with the SLACK_WEBHOOK_URL environment variable. Default: None\n        test_mode (bool, optional): If True, will send a test alert at the first check.\n            Default: False\n    \"\"\"\n\n    def __init__(\n        self,\n        threshold: float = 10,\n        sample_freq: int = 5,\n        window_size: int = 120,\n        wait: int = 120,\n        slack_webhook_url: Optional[str] = None,\n        test_mode: bool = False,\n    ) -> None:\n        self.sample_freq = sample_freq\n        self.window_size = window_size\n        self.wait = wait\n        self.slack_webhook_url = slack_webhook_url\n        self.test_mode = test_mode\n\n        if not self.slack_webhook_url:\n            self.slack_webhook_url = os.environ.get('SLACK_WEBHOOK_URL', None)\n\n        if self.slack_webhook_url:\n            # fail fast if missing import\n            try:\n                import slack_sdk\n                del slack_sdk\n            except ImportError as e:\n                raise MissingConditionalImportError('health_checker', 'slack_sdk', None) from e\n\n        self.last_sample = 0\n        self.last_check = 0\n\n        self.metrics = []\n        if self._is_available():\n            self.metrics.append(GPUUtilization(threshold))\n\n    def init(self, state: State, logger: Logger) -> None:\n        pass\n\n    def after_train_batch(self, state: State, logger: Logger):\n        if not self.metrics:\n            return\n\n        if self._sample(state.timestamp):\n            for metric in self.metrics:\n                metric.sample()\n\n        if self._check(state.timestamp):\n            for metric in self.metrics:\n                message, alert = metric.check()\n                if self.test_mode and message:\n                    alert = True\n                    message = '[**THIS IS A TEST**]' + message\n                if alert and not metric.alerted:\n                    self._alert(message, state)\n                    metric.alerted = True\n                metric.clear()\n\n    def _sample(self, timestamp: Timestamp) -> bool:\n        now = timestamp.total_wct.seconds\n\n        if now < self.wait:\n            return False\n\n        if now - self.last_sample >= self.sample_freq:\n            self.last_sample = now\n            return True\n\n        return False\n\n    def _check(self, timestamp: Timestamp) -> bool:\n        now = timestamp.total_wct.seconds\n\n        if now - self.last_check >= self.window_size:\n            self.last_check = now\n            return True\n        return False\n\n    def _alert(self, message: str, state: State) -> None:\n        prefix = '[{now}][{run_name}][node_rank={node_rank}]'.format(\n            now=datetime.now(),\n            run_name=state.run_name,\n            node_rank=dist.get_node_rank(),\n        )\n\n        node_name = os.environ.get('NODENAME', None)\n        if node_name is not None:\n            prefix += f'[node={node_name}]'\n\n        message = prefix + ' : ' + message\n\n        logging.warning(message)\n        if self.slack_webhook_url:\n            from slack_sdk.webhook import WebhookClient\n            client = WebhookClient(url=self.slack_webhook_url)\n            client.send(text=message)\n\n    @staticmethod\n    def _is_available() -> bool:\n        if not torch.cuda.is_available():\n            return False\n        try:\n            import pynvml\n            pynvml.nvmlInit()  # type: ignore\n            return True\n        except ImportError:\n            raise MissingConditionalImportError('health_checker', 'pynvml', None)\n        except pynvml.NVMLError_LibraryNotFound:  # type: ignore\n            logging.warning('NVML not found, disabling GPU health checking')\n        except Exception as e:\n            logging.warning(f'Error initializing NVML: {e}')\n\n        return False\n\n\nclass GPUUtilization:\n    \"\"\"GPU Utilization Metric.\"\"\"\n\n    def __init__(self, threshold=10) -> None:\n        self.samples = deque()\n        self.threshold = threshold\n        self.alerted = False\n\n    def sample(self) -> None:\n        if dist.get_local_rank() == 0:\n            sample = self._sample()\n            if sample is not None:\n                self.samples.append(sample)\n\n    def _sample(self) -> Optional[List]:\n        try:\n            import pynvml\n        except ImportError:\n            raise MissingConditionalImportError('health_checker', 'pynvml', None)\n\n        try:\n            samples = []\n            device_count = pynvml.nvmlDeviceGetCount()\n            for i in range(device_count):\n                handle = pynvml.nvmlDeviceGetHandleByIndex(i)\n                samples.append(pynvml.nvmlDeviceGetUtilizationRates(handle).gpu)\n        except pynvml.NVMLError:\n            return None\n        return samples\n\n    def check(self) -> Tuple[Optional[str], bool]:\n        if dist.get_local_rank() == 0:\n            average_sample = np.nanmean(list(self.samples), axis=0)\n            if np.nanmax(average_sample) - np.nanmin(average_sample) > self.threshold:\n                message = f'Abnormal GPU utilizations: {average_sample}'\n                return message, True\n            else:\n                message = f':+1: Normal GPU utilizations: {average_sample}'\n                return message, False\n        return None, False\n\n    def clear(self) -> None:\n        self.samples.clear()\n", "path": "composer/callbacks/health_checker.py"}]}