diff --git a/lib/matplotlib/legend.py b/lib/matplotlib/legend.py
index d0590824ad84..ce0eaa056815 100644
--- a/lib/matplotlib/legend.py
+++ b/lib/matplotlib/legend.py
@@ -1173,13 +1173,26 @@ def _get_legend_handles(axs, legend_handler_map=None):
 
 def _get_legend_handles_labels(axs, legend_handler_map=None):
     """Return handles and labels for legend."""
+    log = logging.getLogger(__name__)
     handles = []
     labels = []
+    all_underscored = None
     for handle in _get_legend_handles(axs, legend_handler_map):
         label = handle.get_label()
-        if label and not label.startswith('_'):
-            handles.append(handle)
-            labels.append(label)
+        if label:
+            if not label.startswith('_'):
+                handles.append(handle)
+                labels.append(label)
+                all_underscored = False
+            else:
+                all_underscored = True if all_underscored is not False \
+                    else False
+
+    if all_underscored is True:
+        log.warning("All labels starting with an underscore were "
+                    "detected. Note that artists whose label start with an "
+                    "underscore are ignored when legend() is called with "
+                    "no argument")
     return handles, labels
 
 
@@ -1254,9 +1267,7 @@ def _parse_legend_args(axs, *args, handles=None, labels=None, **kwargs):
         handles, labels = _get_legend_handles_labels(axs, handlers)
         if not handles:
             log.warning(
-                "No artists with labels found to put in legend.  Note that "
-                "artists whose label start with an underscore are ignored "
-                "when legend() is called with no argument.")
+                "No handles with labels found to put in legend.")
 
     # One argument. User defined labels - automatic handle detection.
     elif len(args) == 1: