8000 [cutlass backend] Reduce log level for cutlass runtime error · pytorch/pytorch@bf953f6 · GitHub
[go: up one dir, main page]
Skip to content

Commit bf953f6

Browse files
henrylhtsanghenrylhtsang
committed
[cutlass backend] Reduce log level for cutlass runtime error
Pull Request resolved: #153457 ghstack-source-id: 284547176 @exported-using-ghexport Differential Revision: [D74629230](https://our.internmc.facebook.com/intern/diff/D74629230/)
1 parent bd14409 commit bf953f6

File tree

2 files changed

+34
-5
lines changed

2 files changed

+34
-5
lines changed

‎torch/_inductor/autotune_process.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -718,7 +718,7 @@ def make_run_fn(
718718
workspace_ptr = c_void_p(self.workspace.data_ptr())
719719

720720
# Generate partial function.
721-
return functools.partial(
721+
ret = functools.partial(
722722
run_method,
723723
*args,
724724
*self.extra_args,
@@ -727,6 +727,20 @@ def make_run_fn(
727727
stream_ptr,
728728
)
729729

730+
# sanity check to make sure we cleanup run fn properly
731+
try:
732+
ret()
733+
except RuntimeError as e:
734+
err_msg = str(e)
735+
736+
def raise_runtime_error():
737+
raise RuntimeError(err_msg)
738+
739+
self.cleanup_run_fn()
740+
return raise_runtime_error
741+
742+
return ret
743+
730744
def update_workspace_size(self) -> None:
731745
if self._workspace_size_updated:
732746
return

‎torch/_inductor/select_algorithm.py

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2265,16 +2265,31 @@ def benchmark_choices(
22652265
log.warning("Not yet implemented: %s", e)
22662266
timing = float("inf")
22672267
except RuntimeError as e:
2268+
from torch._inductor.codegen.cuda.cuda_kernel import CUDATemplateCaller
2269+
2270+
if not isinstance(choice, CUDATemplateCaller):
2271+
log.error(
2272+
"CUDA runtime error during autotuning: \n%s. \nIgnoring this choice.",
2273+
e,
2274+
)
22682275
msg = str(e)
22692276
if "invalid argument" in msg:
22702277
msg += "\n\nThis may mean this GPU is too small for max_autotune mode.\n\n"
22712278
else:
22722279
if "illegal memory access" in msg:
22732280
msg += "\n\nEither error in template or triton bug.\n"
2274-
log.error(
2275-
"Runtime error during autotuning: \n%s. \nIgnoring this choice.",
2276-
msg,
2277-
)
2281+
2282+
if isinstance(choice, CUDATemplateCaller):
2283+
log.debug(
2284+
"Runtime error during autotuning: \n%s. \nIgnoring this choice.",
2285+
msg,
2286+
exc_info=True,
2287+
)
2288+
else:
2289+
log.error(
2290+
"Runtime error during autotuning: \n%s. \nIgnoring this choice.",
2291+
msg,
2292+
)
22782293
timing = float("inf")
22792294
except AssertionError as e:
22802295
raise AssertionError( # noqa: B904

0 commit comments

Comments
 (0)
0