diff --git a/jdaviz/core/template_mixin.py b/jdaviz/core/template_mixin.py
index cf490f1952..37b9bba92d 100644
--- a/jdaviz/core/template_mixin.py
+++ b/jdaviz/core/template_mixin.py
@@ -2056,6 +2056,10 @@ def is_not_annulus(subset):
         return super()._is_valid_item(subset, locals())
 
     def _update_subset(self, subset, attribute=None):
+        if attribute == 'label':
+            # We handle this in _rename_subset
+            return
+
         if (attribute == 'subset_state' and
             ((self.is_multiselect and subset.label in self.selected)
              or (subset.label == self.selected))):