it seems that callback can be triggered after unsubscribe (bug in cupti). Defend ourselves from that.
probably due to not calling cuptiFinalize in CUDA 10.1. We will evaluate that.
PiperOrigin-RevId: 267419048
diff --git a/tensorflow/core/profiler/internal/gpu/cupti_tracer.cc b/tensorflow/core/profiler/internal/gpu/cupti_tracer.cc
index 1266856..4c8ace7 100644
--- a/tensorflow/core/profiler/internal/gpu/cupti_tracer.cc
+++ b/tensorflow/core/profiler/internal/gpu/cupti_tracer.cc
@@ -755,6 +755,7 @@
Status CuptiTracer::HandleCallback(CUpti_CallbackDomain domain,
CUpti_CallbackId cbid,
const CUpti_CallbackData *callback_info) {
+ if (!api_tracing_enabled_) return Status::OK(); // already unsubscribed.
if (domain != CUPTI_CB_DOMAIN_DRIVER_API) return Status::OK();
if (callback_info->callbackSite == CUPTI_API_ENTER) {
// Stash away the current Cupti timestamp into callback_info.
