feat: added recursive mapping

ref: N25B-400

.gitignore

@@ -273,8 +273,6 @@ experiment-*.log
 
 
 
-
-
 
 
 

src/control_backend/agents/actuation/robot_gesture_agent.py

@@ -1,4 +1,5 @@
 import json
+import logging
 
 import zmq
 import zmq.asyncio as azmq
@@ -8,6 +9,8 @@ from control_backend.core.agent_system import InternalMessage
 from control_backend.core.config import settings
 from control_backend.schemas.ri_message import GestureCommand, RIEndpoint
 
+experiment_logger = logging.getLogger(settings.logging_settings.experiment_logger_name)
+
 
 class RobotGestureAgent(BaseAgent):
     """
@@ -111,6 +114,7 @@ class RobotGestureAgent(BaseAgent):
                         gesture_command.data,
                     )
                     return
+            experiment_logger.action("Gesture: %s", gesture_command.data)
             await self.pubsocket.send_json(gesture_command.model_dump())
         except Exception:
             self.logger.exception("Error processing internal message.")

src/control_backend/agents/bdi/bdi_core_agent.py

@@ -1,6 +1,7 @@
 import asyncio
 import copy
 import json
+import logging
 import time
 from collections.abc import Iterable
 
@@ -19,6 +20,9 @@ from control_backend.schemas.ri_message import GestureCommand, RIEndpoint, Speec
 DELIMITER = ";\n"  # TODO: temporary until we support lists in AgentSpeak
 
 
+experiment_logger = logging.getLogger(settings.logging_settings.experiment_logger_name)
+
+
 class BDICoreAgent(BaseAgent):
     """
     BDI Core Agent.
@@ -207,6 +211,9 @@ class BDICoreAgent(BaseAgent):
         else:
             term = agentspeak.Literal(name)
 
+        if name != "user_said":
+            experiment_logger.observation(f"Formed new belief: {name}{f'={args}' if args else ''}")
+
         self.bdi_agent.call(
             agentspeak.Trigger.addition,
             agentspeak.GoalType.belief,
@@ -244,6 +251,9 @@ class BDICoreAgent(BaseAgent):
             new_args = (agentspeak.Literal(arg) for arg in args)
             term = agentspeak.Literal(name, new_args)
 
+        if name != "user_said":
+            experiment_logger.observation(f"Removed belief: {name}{f'={args}' if args else ''}")
+
         result = self.bdi_agent.call(
             agentspeak.Trigger.removal,
             agentspeak.GoalType.belief,
@@ -386,6 +396,8 @@ class BDICoreAgent(BaseAgent):
                 body=str(message_text),
             )
 
+            experiment_logger.chat(str(message_text), extra={"role": "assistant"})
+
             self.add_behavior(self.send(chat_history_message))
 
             yield
@@ -441,6 +453,7 @@ class BDICoreAgent(BaseAgent):
             trigger_name = agentspeak.grounded(term.args[0], intention.scope)
 
             self.logger.debug("Started trigger %s", trigger_name)
+            experiment_logger.observation("Triggered: %s", trigger_name)
 
             msg = InternalMessage(
                 to=settings.agent_settings.user_interrupt_name,

src/control_backend/agents/communication/ri_communication_agent.py

@@ -55,6 +55,7 @@ class RICommunicationAgent(BaseAgent):
         self.connected = False
         self.gesture_agent: RobotGestureAgent | None = None
         self.speech_agent: RobotSpeechAgent | None = None
+        self.visual_emotion_recognition_agent: VisualEmotionRecognitionAgent | None = None
 
     async def setup(self):
         """
@@ -218,6 +219,7 @@ class RICommunicationAgent(BaseAgent):
                         socket_address=addr,
                         bind=bind,
                     )
+                    self.visual_emotion_recognition_agent = visual_emotion_agent
                     await visual_emotion_agent.start()
                 case _:
                     self.logger.warning("Unhandled negotiation id: %s", id)
@@ -323,6 +325,9 @@ class RICommunicationAgent(BaseAgent):
 
         if self.speech_agent is not None:
             await self.speech_agent.stop()
+        
+        if self.visual_emotion_recognition_agent is not None:
+            await self.visual_emotion_recognition_agent.stop()
 
         if self.pub_socket is not None:
             self.pub_socket.close()
@@ -332,6 +337,7 @@ class RICommunicationAgent(BaseAgent):
             self.connected = True
 
     async def handle_message(self, msg: InternalMessage):
+        return 
         try:
             pause_command = PauseCommand.model_validate_json(msg.body)
             await self._req_socket.send_json(pause_command.model_dump())

src/control_backend/agents/llm/llm_agent.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import logging
 import re
 import uuid
 from collections.abc import AsyncGenerator
@@ -14,6 +15,8 @@ from control_backend.core.config import settings
 from ...schemas.llm_prompt_message import LLMPromptMessage
 from .llm_instructions import LLMInstructions
 
+experiment_logger = logging.getLogger(settings.logging_settings.experiment_logger_name)
+
 
 class LLMAgent(BaseAgent):
     """
@@ -170,7 +173,7 @@ class LLMAgent(BaseAgent):
             *self.history,
         ]
 
-        message_id = str(uuid.uuid4())  # noqa
+        message_id = str(uuid.uuid4())
 
         try:
             full_message = ""
@@ -179,10 +182,9 @@ class LLMAgent(BaseAgent):
                 full_message += token
                 current_chunk += token
 
-                self.logger.llm(
-                    "Received token: %s",
+                experiment_logger.chat(
                     full_message,
-                    extra={"reference": message_id},  # Used in the UI to update old logs
+                    extra={"role": "assistant", "reference": message_id, "partial": True},
                 )
 
                 # Stream the message in chunks separated by punctuation.
@@ -197,6 +199,11 @@ class LLMAgent(BaseAgent):
             # Yield any remaining tail
             if current_chunk:
                 yield current_chunk
+
+            experiment_logger.chat(
+                full_message,
+                extra={"role": "assistant", "reference": message_id, "partial": False},
+            )
         except httpx.HTTPError as err:
             self.logger.error("HTTP error.", exc_info=err)
             yield "LLM service unavailable."
@@ -212,7 +219,7 @@ class LLMAgent(BaseAgent):
         :yield: Raw text tokens (deltas) from the SSE stream.
         :raises httpx.HTTPError: If the API returns a non-200 status.
         """
-        async with httpx.AsyncClient() as client:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(20.0)) as client:
             async with client.stream(
                 "POST",
                 settings.llm_settings.local_llm_url,

src/control_backend/agents/perception/transcription_agent/transcription_agent.py

@@ -1,4 +1,5 @@
 import asyncio
+import logging
 
 import numpy as np
 import zmq
@@ -10,6 +11,8 @@ from control_backend.core.config import settings
 
 from .speech_recognizer import SpeechRecognizer
 
+experiment_logger = logging.getLogger(settings.logging_settings.experiment_logger_name)
+
 
 class TranscriptionAgent(BaseAgent):
     """
@@ -25,6 +28,8 @@ class TranscriptionAgent(BaseAgent):
     :ivar audio_in_socket: The ZMQ SUB socket instance.
     :ivar speech_recognizer: The speech recognition engine instance.
     :ivar _concurrency: Semaphore to limit concurrent transcriptions.
+    :ivar _current_speech_reference: The reference of the current user utterance, for synchronising
+        experiment logs.
     """
 
     def __init__(self, audio_in_address: str):
@@ -39,6 +44,7 @@ class TranscriptionAgent(BaseAgent):
         self.audio_in_socket: azmq.Socket | None = None
         self.speech_recognizer = None
         self._concurrency = None
+        self._current_speech_reference: str | None = None
 
     async def setup(self):
         """
@@ -63,6 +69,10 @@ class TranscriptionAgent(BaseAgent):
 
         self.logger.info("Finished setting up %s", self.name)
 
+    async def handle_message(self, msg: InternalMessage):
+        if msg.thread == "voice_activity":
+            self._current_speech_reference = msg.body
+
     async def stop(self):
         """
         Stop the agent and close sockets.
@@ -96,24 +106,25 @@ class TranscriptionAgent(BaseAgent):
 
     async def _share_transcription(self, transcription: str):
         """
-        Share a transcription to the other agents that depend on it.
+        Share a transcription to the other agents that depend on it, and to experiment logs.
 
         Currently sends to:
         - :attr:`settings.agent_settings.text_belief_extractor_name`
+        - The UI via the experiment logger
 
         :param transcription: The transcribed text.
         """
-        receiver_names = [
-            settings.agent_settings.text_belief_extractor_name,
-        ]
+        experiment_logger.chat(
+            transcription,
+            extra={"role": "user", "reference": self._current_speech_reference, "partial": False},
+        )
 
-        for receiver_name in receiver_names:
-            message = InternalMessage(
-                to=receiver_name,
-                sender=self.name,
-                body=transcription,
-            )
-            await self.send(message)
+        message = InternalMessage(
+            to=settings.agent_settings.text_belief_extractor_name,
+            sender=self.name,
+            body=transcription,
+        )
+        await self.send(message)
 
     async def _transcribing_loop(self) -> None:
         """
@@ -129,10 +140,9 @@ class TranscriptionAgent(BaseAgent):
                 audio = np.frombuffer(audio_data, dtype=np.float32)
                 speech = await self._transcribe(audio)
                 if not speech:
-                    self.logger.info("Nothing transcribed.")
+                    self.logger.debug("Nothing transcribed.")
                     continue
 
-                self.logger.info("Transcribed speech: %s", speech)
                 await self._share_transcription(speech)
             except Exception as e:
                 self.logger.error(f"Error in transcription loop: {e}")

src/control_backend/agents/perception/vad_agent.py

@@ -1,4 +1,6 @@
 import asyncio
+import logging
+import uuid
 
 import numpy as np
 import torch
@@ -12,6 +14,8 @@ from control_backend.schemas.internal_message import InternalMessage
 from ...schemas.program_status import PROGRAM_STATUS, ProgramStatus
 from .transcription_agent.transcription_agent import TranscriptionAgent
 
+experiment_logger = logging.getLogger(settings.logging_settings.experiment_logger_name)
+
 
 class SocketPoller[T]:
     """
@@ -252,6 +256,18 @@ class VADAgent(BaseAgent):
             if prob > prob_threshold:
                 if self.i_since_speech > non_speech_patience + begin_silence_length:
                     self.logger.debug("Speech started.")
+                    reference = str(uuid.uuid4())
+                    experiment_logger.chat(
+                        "...",
+                        extra={"role": "user", "reference": reference, "partial": True},
+                    )
+                    await self.send(
+                        InternalMessage(
+                            to=settings.agent_settings.transcription_name,
+                            body=reference,
+                            thread="voice_activity",
+                        )
+                    )
                 self.audio_buffer = np.append(self.audio_buffer, chunk)
                 self.i_since_speech = 0
                 continue

src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py

@@ -97,6 +97,7 @@ class VisualEmotionRecognitionAgent(BaseAgent):
 
                 if frame_image is None:
                     # Could not decode image, skip this frame
+                    self.logger.warning("Received invalid video frame, skipping.")
                     continue
 
                 # Get the dominant emotion from each face

src/control_backend/agents/user_interrupt/user_interrupt_agent.py

@@ -1,4 +1,5 @@
 import json
+import logging
 
 import zmq
 from zmq.asyncio import Context
@@ -8,7 +9,7 @@ from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator
 from control_backend.core.agent_system import InternalMessage
 from control_backend.core.config import settings
 from control_backend.schemas.belief_message import Belief, BeliefMessage
-from control_backend.schemas.program import ConditionalNorm, Program
+from control_backend.schemas.program import ConditionalNorm, Goal, Program
 from control_backend.schemas.ri_message import (
     GestureCommand,
     PauseCommand,
@@ -16,6 +17,8 @@ from control_backend.schemas.ri_message import (
     SpeechCommand,
 )
 
+experiment_logger = logging.getLogger(settings.logging_settings.experiment_logger_name)
+
 
 class UserInterruptAgent(BaseAgent):
     """
@@ -194,6 +197,7 @@ class UserInterruptAgent(BaseAgent):
             case "transition_phase":
                 new_phase_id = msg.body
                 self.logger.info(f"Phase transition detected: {new_phase_id}")
+                experiment_logger.observation("Transitioned to next phase.")
 
                 payload = {"type": "phase_update", "id": new_phase_id}
 
@@ -246,6 +250,18 @@ class UserInterruptAgent(BaseAgent):
             self._cond_norm_map = {}
             self._cond_norm_reverse_map = {}
 
+            def _register_goal(goal: Goal):
+                """Recursively register goals and their subgoals."""
+                slug = AgentSpeakGenerator.slugify(goal)
+                self._goal_map[str(goal.id)] = slug
+                self._goal_reverse_map[slug] = str(goal.id)
+
+                # Recursively check steps for subgoals
+                if goal.plan and goal.plan.steps:
+                    for step in goal.plan.steps:
+                        if isinstance(step, Goal):
+                            _register_goal(step)
+
             for phase in program.phases:
                 for trigger in phase.triggers:
                     slug = AgentSpeakGenerator.slugify(trigger)
@@ -253,8 +269,7 @@ class UserInterruptAgent(BaseAgent):
                     self._trigger_reverse_map[slug] = str(trigger.id)
 
                 for goal in phase.goals:
-                    self._goal_map[str(goal.id)] = AgentSpeakGenerator.slugify(goal)
-                    self._goal_reverse_map[AgentSpeakGenerator.slugify(goal)] = str(goal.id)
+                    _register_goal(goal)
 
                 for goal, id in self._goal_reverse_map.items():
                     self.logger.debug(f"Goal mapping: UI ID {goal} -> {id}")
@@ -296,6 +311,7 @@ class UserInterruptAgent(BaseAgent):
 
         :param text_to_say: The string that the robot has to say.
         """
+        experiment_logger.chat(text_to_say, extra={"role": "user"})
         cmd = SpeechCommand(data=text_to_say, is_priority=True)
         out_msg = InternalMessage(
             to=settings.agent_settings.robot_speech_name,

src/control_backend/logging/dated_file_handler.py

@@ -12,21 +12,12 @@ class DatedFileHandler(FileHandler):
         super().__init__(**kwargs)
 
     def _make_filename(self) -> str:
-        """
-        Create the filename for the current logfile, using the configured file prefix and the
-        current date and time. If the directory does not exist, it gets created.
-
-        :return: A filepath.
-        """
         filepath = Path(f"{self._file_prefix}-{datetime.now():%Y%m%d-%H%M%S}.log")
         if not filepath.parent.is_dir():
             filepath.parent.mkdir(parents=True, exist_ok=True)
         return str(filepath)
 
     def do_rollover(self):
-        """
-        Close the current logfile and create a new one with the current date and time.
-        """
         self.acquire()
         try:
             if self.stream: