diff --git a/src/control_backend/agents/vad_agent.py b/src/control_backend/agents/vad_agent.py
index 4fef563..7b87fbb 100644
--- a/src/control_backend/agents/vad_agent.py
+++ b/src/control_backend/agents/vad_agent.py
@@ -58,8 +58,10 @@ class Streaming(CyclicBehaviour):
     async def run(self) -> None:
         data = await self.audio_in_poller.poll()
         if data is None:
-            logger.debug("No audio data received. Discarding buffer until new data arrives.")
-            self.audio_buffer = np.array([], dtype=np.float32)
+            if len(self.audio_buffer) > 0:
+                logger.debug("No audio data received. Discarding buffer until new data arrives.")
+                self.audio_buffer = np.array([], dtype=np.float32)
+                self.i_since_speech = 100
             return
 
         # copy otherwise Torch will be sad that it's immutable