diff --git a/custom_components/adaptive_lighting/switch.py b/custom_components/adaptive_lighting/switch.py
index 772aa1e0..86f430bc 100644
--- a/custom_components/adaptive_lighting/switch.py
+++ b/custom_components/adaptive_lighting/switch.py
@@ -1465,7 +1465,8 @@ class AdaptiveSwitch(SwitchEntity, RestoreEntity):
     async def _respond_to_off_to_on_event(self, entity_id: str, event: Event) -> None:
         assert not self.manager.is_proactively_adapting(event.context.id)
         if (
-            not self._detect_non_ha_changes
+            self._take_over_control
+            and not self._detect_non_ha_changes
             and not self.manager._off_to_on_state_event_is_from_turn_on(
                 entity_id,
                 event,