From ca40b0fcc632eb9962756f16e4262e239d49f0bb Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Feb 2020 11:32:44 -0800 Subject: [PATCH] [tune][minor] Avoid throwing error when gpu check fails (#7362) --- python/ray/tune/utils/util.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/ray/tune/utils/util.py b/python/ray/tune/utils/util.py index 934487cfd..7e6e8125e 100644 --- a/python/ray/tune/utils/util.py +++ b/python/ray/tune/utils/util.py @@ -58,7 +58,12 @@ class UtilMonitor(Thread): self.values["ram_util_percent"].append( float(getattr(psutil.virtual_memory(), "percent"))) if GPUtil is not None: - for gpu in GPUtil.getGPUs(): + gpu_list = [] + try: + gpu_list = GPUtil.getGPUs() + except Exception: + logger.debug("GPUtil failed to retrieve GPUs.") + for gpu in gpu_list: self.values["gpu_util_percent" + str(gpu.id)].append( float(gpu.load)) self.values["vram_util_percent" + str(gpu.id)].append(