pepperplus/pepperplus-cb
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into refactor/config-file

Browse Source 
# Conflicts:
#	src/control_backend/agents/ri_communication_agent.py
#	src/control_backend/core/config.py
#	src/control_backend/main.py
This commit is contained in:
Twirre Meulenbelt
2025-11-19 17:30:48 +01:00
parent 93b8db03e7 9963134262
commit 64db25c974
46 changed files with 1207 additions and 651 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/control_backend/agents/__init__.py
View File

@@ -1,7 +1 @@
from .base import BaseAgent as BaseAgent
from .belief_collector.belief_collector import BeliefCollectorAgent as BeliefCollectorAgent
from .llm.llm import LLMAgent as LLMAgent
from .ri_command_agent import RICommandAgent as RICommandAgent
from .ri_communication_agent import RICommunicationAgent as RICommunicationAgent
from .transcription.transcription_agent import TranscriptionAgent as TranscriptionAgent
from .vad_agent import VADAgent as VADAgent

1
src/control_backend/agents/actuation/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .robot_speech_agent import RobotSpeechAgent as RobotSpeechAgent

14
src/control_backend/agents/ri_command_agent.py → src/control_backend/agents/actuation/robot_speech_agent.py
View File

@@ -10,7 +10,7 @@ from control_backend.core.config import settings
from control_backend.schemas.ri_message import SpeechCommand
class RICommandAgent(BaseAgent):
class RobotSpeechAgent(BaseAgent):
subsocket: zmq.Socket
pubsocket: zmq.Socket
address = ""
@@ -29,7 +29,7 @@ class RICommandAgent(BaseAgent):
self.address = address
self.bind = bind
class SendCommandsBehaviour(CyclicBehaviour):
class SendZMQCommandsBehaviour(CyclicBehaviour):
"""Behaviour for sending commands received from the UI."""
async def run(self):
@@ -50,7 +50,7 @@ class RICommandAgent(BaseAgent):
except Exception as e:
self.agent.logger.error("Error processing message: %s", e)
class SendPythonCommandsBehaviour(CyclicBehaviour):
class SendSpadeCommandsBehaviour(CyclicBehaviour):
"""Behaviour for sending commands received from other Python agents."""
async def run(self):
@@ -64,7 +64,7 @@ class RICommandAgent(BaseAgent):
async def setup(self):
"""
Setup the command agent
Setup the robot speech command agent
"""
self.logger.info("Setting up %s", self.jid)
@@ -72,7 +72,7 @@ class RICommandAgent(BaseAgent):
# To the robot
self.pubsocket = context.socket(zmq.PUB)
if self.bind:
if self.bind: # TODO: Should this ever be the case?
self.pubsocket.bind(self.address)
else:
self.pubsocket.connect(self.address)
@@ -83,8 +83,8 @@ class RICommandAgent(BaseAgent):
self.subsocket.setsockopt(zmq.SUBSCRIBE, b"command")
# Add behaviour to our agent
commands_behaviour = self.SendCommandsBehaviour()
commands_behaviour = self.SendZMQCommandsBehaviour()
self.add_behaviour(commands_behaviour)
self.add_behaviour(self.SendPythonCommandsBehaviour())
self.add_behaviour(self.SendSpadeCommandsBehaviour())
self.logger.info("Finished setting up %s", self.jid)

9
src/control_backend/agents/bdi/__init__.py
View File

@@ -1,2 +1,7 @@
from .bdi_core import BDICoreAgent as BDICoreAgent
from .text_extractor import TBeliefExtractorAgent as TBeliefExtractorAgent
from .bdi_core_agent.bdi_core_agent import BDICoreAgent as BDICoreAgent
from .belief_collector_agent.belief_collector_agent import (
BDIBeliefCollectorAgent as BDIBeliefCollectorAgent,
)
from .text_belief_extractor_agent.text_belief_extractor_agent import (
TextBeliefExtractorAgent as TextBeliefExtractorAgent,
)

4
src/control_backend/agents/bdi/bdi_core.py → src/control_backend/agents/bdi/bdi_core_agent/bdi_core_agent.py
View File

@@ -7,7 +7,7 @@ from spade_bdi.bdi import BDIAgent
from control_backend.core.config import settings
from .behaviours.belief_setter import BeliefSetterBehaviour
from .behaviours.belief_setter_behaviour import BeliefSetterBehaviour
from .behaviours.receive_llm_resp_behaviour import ReceiveLLMResponseBehaviour
@@ -57,7 +57,7 @@ class BDICoreAgent(BDIAgent):
class SendBehaviour(OneShotBehaviour):
async def run(self) -> None:
msg = Message(
to=settings.agent_settings.llm_agent_name + "@" + settings.agent_settings.host,
to=settings.agent_settings.llm_name + "@" + settings.agent_settings.host,
body=text,
)

2
src/control_backend/agents/bdi/behaviours/belief_setter.py → src/control_backend/agents/bdi/bdi_core_agent/behaviours/belief_setter_behaviour.py
View File

@@ -32,7 +32,7 @@ class BeliefSetterBehaviour(CyclicBehaviour):
self.agent.logger.debug("Processing message from sender: %s", sender)
match sender:
case settings.agent_settings.belief_collector_agent_name:
case settings.agent_settings.bdi_belief_collector_name:
self.agent.logger.debug(
"Message is from the belief collector agent. Processing as belief message."
)

4
src/control_backend/agents/bdi/behaviours/receive_llm_resp_behaviour.py → src/control_backend/agents/bdi/bdi_core_agent/behaviours/receive_llm_resp_behaviour.py
View File

@@ -15,14 +15,14 @@ class ReceiveLLMResponseBehaviour(CyclicBehaviour):
sender = msg.sender.node
match sender:
case settings.agent_settings.llm_agent_name:
case settings.agent_settings.llm_name:
content = msg.body
self.agent.logger.info("Received LLM response: %s", content)
speech_command = SpeechCommand(data=content)
message = Message(
to=settings.agent_settings.ri_command_agent_name
to=settings.agent_settings.robot_speech_name
+ "@"
+ settings.agent_settings.host,
sender=self.agent.jid,

0
src/control_backend/agents/bdi/rules.asl → src/control_backend/agents/bdi/bdi_core_agent/rules.asl
View File

6
src/control_backend/agents/belief_collector/behaviours/continuous_collect.py → src/control_backend/agents/bdi/belief_collector_agent/behaviours/belief_collector_behaviour.py
View File

@@ -7,7 +7,7 @@ from spade.behaviour import CyclicBehaviour
from control_backend.core.config import settings
class ContinuousBeliefCollector(CyclicBehaviour):
class BeliefCollectorBehaviour(CyclicBehaviour):
"""
Continuously collects beliefs/emotions from extractor agents:
Then we send a unified belief packet to the BDI agent.
@@ -35,7 +35,7 @@ class ContinuousBeliefCollector(CyclicBehaviour):
msg_type = payload.get("type")
# Prefer explicit 'type' field
if msg_type == "belief_extraction_text" or sender_node == "belief_text_agent_mock":
if msg_type == "belief_extraction_text" or sender_node == "bel_text_agent_mock":
self.agent.logger.debug(
"Message routed to _handle_belief_text (sender=%s)", sender_node
)
@@ -83,7 +83,7 @@ class ContinuousBeliefCollector(CyclicBehaviour):
if not beliefs:
return
to_jid = f"{settings.agent_settings.bdi_core_agent_name}@{settings.agent_settings.host}"
to_jid = f"{settings.agent_settings.bdi_core_name}@{settings.agent_settings.host}"
msg = Message(to=to_jid, sender=self.agent.jid, thread="beliefs")
msg.body = json.dumps(beliefs)

11
src/control_backend/agents/bdi/belief_collector_agent/belief_collector_agent.py Normal file
View File

@@ -0,0 +1,11 @@
from control_backend.agents.base import BaseAgent
from .behaviours.belief_collector_behaviour import BeliefCollectorBehaviour
class BDIBeliefCollectorAgent(BaseAgent):
async def setup(self):
self.logger.info("BDIBeliefCollectorAgent starting (%s)", self.jid)
# Attach the continuous collector behaviour (listens and forwards to BDI)
self.add_behaviour(BeliefCollectorBehaviour())
self.logger.info("BDIBeliefCollectorAgent ready.")

8
src/control_backend/agents/bdi/behaviours/text_belief_extractor.py → src/control_backend/agents/bdi/text_belief_extractor_agent/behaviours/text_belief_extractor_behaviour.py
View File

@@ -7,7 +7,7 @@ from spade.message import Message
from control_backend.core.config import settings
class BeliefFromText(CyclicBehaviour):
class TextBeliefExtractorBehaviour(CyclicBehaviour):
logger = logging.getLogger(__name__)
# TODO: LLM prompt nog hardcoded
@@ -44,7 +44,7 @@ class BeliefFromText(CyclicBehaviour):
sender = msg.sender.node
match sender:
case settings.agent_settings.transcription_agent_name:
case settings.agent_settings.transcription_name:
self.logger.debug("Received text from transcriber: %s", msg.body)
await self._process_transcription_demo(msg.body)
case _:
@@ -71,7 +71,7 @@ class BeliefFromText(CyclicBehaviour):
belief_message = Message()
belief_message.to = (
settings.agent_settings.belief_collector_agent_name
settings.agent_settings.bdi_belief_collector_name
+ "@"
+ settings.agent_settings.host
)
@@ -95,7 +95,7 @@ class BeliefFromText(CyclicBehaviour):
belief_msg = Message()
belief_msg.to = (
settings.agent_settings.belief_collector_agent_name + "@" + settings.agent_settings.host
settings.agent_settings.bdi_belief_collector_name + "@" + settings.agent_settings.host
)
belief_msg.body = payload
belief_msg.thread = "beliefs"

8
src/control_backend/agents/bdi/text_belief_extractor_agent/text_belief_extractor_agent.py Normal file
View File

@@ -0,0 +1,8 @@
from control_backend.agents.base import BaseAgent
from .behaviours.text_belief_extractor_behaviour import TextBeliefExtractorBehaviour
class TextBeliefExtractorAgent(BaseAgent):
async def setup(self):
self.add_behaviour(TextBeliefExtractorBehaviour())

8
src/control_backend/agents/bdi/text_extractor.py
View File

@@ -1,8 +0,0 @@
from control_backend.agents.base import BaseAgent
from .behaviours.text_belief_extractor import BeliefFromText
class TBeliefExtractorAgent(BaseAgent):
async def setup(self):
self.add_behaviour(BeliefFromText())

11
src/control_backend/agents/belief_collector/belief_collector.py
View File

@@ -1,11 +0,0 @@
from control_backend.agents.base import BaseAgent
from .behaviours.continuous_collect import ContinuousBeliefCollector
class BeliefCollectorAgent(BaseAgent):
async def setup(self):
self.logger.info("BeliefCollectorAgent starting (%s)", self.jid)
# Attach the continuous collector behaviour (listens and forwards to BDI)
self.add_behaviour(ContinuousBeliefCollector())
self.logger.info("BeliefCollectorAgent ready.")

1
src/control_backend/agents/communication/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .ri_communication_agent import RICommunicationAgent as RICommunicationAgent

250
src/control_backend/agents/communication/ri_communication_agent.py Normal file
View File

@@ -0,0 +1,250 @@
import asyncio
import json
import zmq.asyncio
from spade.behaviour import CyclicBehaviour
from zmq.asyncio import Context
from control_backend.agents import BaseAgent
from control_backend.core.config import settings
from ..actuation.robot_speech_agent import RobotSpeechAgent
class RICommunicationAgent(BaseAgent):
req_socket: zmq.Socket
_address = ""
_bind = True
connected = False
def __init__(
self,
jid: str,
password: str,
port: int = settings.agent_settings.default_spade_port,
verify_security: bool = False,
address=settings.zmq_settings.ri_command_address,
bind=False,
):
super().__init__(jid, password, port, verify_security)
self._address = address
self._bind = bind
self._req_socket: zmq.asyncio.Socket | None = None
self.pub_socket: zmq.asyncio.Socket | None = None
class ListenBehaviour(CyclicBehaviour):
async def run(self):
"""
Run the listening (ping) loop indefinetely.
"""
assert self.agent is not None
if not self.agent.connected:
await asyncio.sleep(settings.behaviour_settings.sleep_s)
return
# We need to listen and sent pings.
message = {"endpoint": "ping", "data": {"id": "e.g. some reference id"}}
seconds_to_wait_total = settings.behaviour_settings.sleep_s
try:
await asyncio.wait_for(
self.agent._req_socket.send_json(message), timeout=seconds_to_wait_total / 2
)
except TimeoutError:
self.agent.logger.debug(
"Waited too long to send message - "
"we probably dont have any receivers... but let's check!"
)
# Wait up to {seconds_to_wait_total/2} seconds for a reply
try:
message = await asyncio.wait_for(
self.agent._req_socket.recv_json(), timeout=seconds_to_wait_total / 2
)
# We didnt get a reply
except TimeoutError:
self.agent.logger.info(
f"No ping retrieved in {seconds_to_wait_total} seconds, "
"sending UI disconnection event and attempting to restart."
)
# Make sure we dont retry receiving messages untill we're setup.
self.agent.connected = False
self.agent.remove_behaviour(self)
# Tell UI we're disconnected.
topic = b"ping"
data = json.dumps(False).encode()
if self.agent.pub_socket is None:
self.agent.logger.warning(
"Communication agent pub socket not correctly initialized."
)
else:
try:
await asyncio.wait_for(
self.agent.pub_socket.send_multipart([topic, data]), 5
)
except TimeoutError:
self.agent.logger.warning(
f"Initial connection ping for router timed out in {self.agent.name}."
)
# Try to reboot.
self.agent.logger.debug("Restarting communication agent.")
await self.agent.setup()
self.agent.logger.debug(f'Received message "{message}" from RI.')
if "endpoint" not in message:
self.agent.logger.warning(
"No received endpoint in message, expected ping endpoint."
)
return
# See what endpoint we received
match message["endpoint"]:
case "ping":
topic = b"ping"
data = json.dumps(True).encode()
if self.agent.pub_socket is not None:
await self.agent.pub_socket.send_multipart([topic, data])
await asyncio.sleep(settings.behaviour_settings.sleep_s)
case _:
self.agent.logger.debug(
"Received message with topic different than ping, while ping expected."
)
async def setup_sockets(self, force=False):
"""
Sets up request socket for communication agent.
"""
# Bind request socket
if self._req_socket is None or force:
self._req_socket = Context.instance().socket(zmq.REQ)
if self._bind:
self._req_socket.bind(self._address)
else:
self._req_socket.connect(self._address)
if self.pub_socket is None or force:
self.pub_socket = Context.instance().socket(zmq.PUB)
self.pub_socket.connect(settings.zmq_settings.internal_pub_address)
async def setup(self, max_retries: int = settings.behaviour_settings.comm_setup_max_retries):
"""
Try to set up the communication agent, we have `behaviour_settings.comm_setup_max_retries`
retries in case we don't have a response yet.
"""
self.logger.info("Setting up %s", self.jid)
# Bind request socket
await self.setup_sockets()
retries = 0
# Let's try a certain amount of times before failing connection
while retries < max_retries:
# Make sure the socket is properly setup.
if self._req_socket is None:
continue
# Send our message and receive one back
message = {"endpoint": "negotiate/ports", "data": {}}
await self._req_socket.send_json(message)
retry_frequency = 1.0
try:
received_message = await asyncio.wait_for(
self._req_socket.recv_json(), timeout=retry_frequency
)
except TimeoutError:
self.logger.warning(
"No connection established in %d seconds (attempt %d/%d)",
retries * retry_frequency,
retries + 1,
max_retries,
)
retries += 1
continue
except Exception as e:
self.logger.warning("Unexpected error during negotiation: %s", e)
retries += 1
continue
# Validate endpoint
endpoint = received_message.get("endpoint")
if endpoint != "negotiate/ports":
self.logger.warning(
"Invalid endpoint '%s' received (attempt %d/%d)",
endpoint,
retries + 1,
max_retries,
)
retries += 1
await asyncio.sleep(1)
continue
# At this point, we have a valid response
try:
for port_data in received_message["data"]:
id = port_data["id"]
port = port_data["port"]
bind = port_data["bind"]
if not bind:
addr = f"tcp://localhost:{port}"
else:
addr = f"tcp://*:{port}"
match id:
case "main":
if addr != self._address:
if not bind:
self._req_socket.connect(addr)
else:
self._req_socket.bind(addr)
case "actuation":
ri_commands_agent = RobotSpeechAgent(
settings.agent_settings.robot_speech_name
+ "@"
+ settings.agent_settings.host,
settings.agent_settings.robot_speech_name,
address=addr,
bind=bind,
)
await ri_commands_agent.start()
case _:
self.logger.warning("Unhandled negotiation id: %s", id)
except Exception as e:
self.logger.warning("Error unpacking negotiation data: %s", e)
retries += 1
await asyncio.sleep(1)
continue
# setup succeeded
break
else:
self.logger.warning("Failed to set up %s after %d retries", self.name, max_retries)
return
# Set up ping behaviour
listen_behaviour = self.ListenBehaviour()
self.add_behaviour(listen_behaviour)
# Let UI know that we're connected
topic = b"ping"
data = json.dumps(True).encode()
if self.pub_socket is None:
self.logger.warning("Communication agent pub socket not correctly initialized.")
else:
try:
await asyncio.wait_for(self.pub_socket.send_multipart([topic, data]), 5)
except TimeoutError:
self.logger.warning("Initial connection ping for router timed out in com_ri_agent.")
# Make sure to start listening now that we're connected.
self.connected = True
self.logger.info("Finished setting up %s", self.jid)

1
src/control_backend/agents/llm/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .llm_agent import LLMAgent as LLMAgent

4
src/control_backend/agents/llm/llm.py → src/control_backend/agents/llm/llm_agent.py
View File

@@ -39,7 +39,7 @@ class LLMAgent(BaseAgent):
sender,
)
if sender == settings.agent_settings.bdi_core_agent_name:
if sender == settings.agent_settings.bdi_core_name:
self.agent.logger.debug("Processing message from BDI Core Agent")
await self._process_bdi_message(msg)
else:
@@ -63,7 +63,7 @@ class LLMAgent(BaseAgent):
Sends a response message back to the BDI Core Agent.
"""
reply = Message(
to=settings.agent_settings.bdi_core_agent_name + "@" + settings.agent_settings.host,
to=settings.agent_settings.bdi_core_name + "@" + settings.agent_settings.host,
body=msg,
)
await self.send(reply)

4
src/control_backend/agents/perception/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
from .transcription_agent.transcription_agent import (
TranscriptionAgent as TranscriptionAgent,
)
from .vad_agent import VADAgent as VADAgent

0
src/control_backend/agents/transcription/speech_recognizer.py → src/control_backend/agents/perception/transcription_agent/speech_recognizer.py
View File

10
src/control_backend/agents/transcription/transcription_agent.py → src/control_backend/agents/perception/transcription_agent/transcription_agent.py
View File

@@ -19,13 +19,13 @@ class TranscriptionAgent(BaseAgent):
"""
def __init__(self, audio_in_address: str):
jid = settings.agent_settings.transcription_agent_name + "@" + settings.agent_settings.host
super().__init__(jid, settings.agent_settings.transcription_agent_name)
jid = settings.agent_settings.transcription_name + "@" + settings.agent_settings.host
super().__init__(jid, settings.agent_settings.transcription_name)
self.audio_in_address = audio_in_address
self.audio_in_socket: azmq.Socket | None = None
class Transcribing(CyclicBehaviour):
class TranscribingBehaviour(CyclicBehaviour):
def __init__(self, audio_in_socket: azmq.Socket):
super().__init__()
max_concurrent_tasks = settings.behaviour_settings.transcription_max_concurrent_tasks
@@ -44,7 +44,7 @@ class TranscriptionAgent(BaseAgent):
async def _share_transcription(self, transcription: str):
"""Share a transcription to the other agents that depend on it."""
receiver_jids = [
settings.agent_settings.text_belief_extractor_agent_name
settings.agent_settings.text_belief_extractor_name
+ "@"
+ settings.agent_settings.host,
] # Set message receivers here
@@ -80,7 +80,7 @@ class TranscriptionAgent(BaseAgent):
self._connect_audio_in_socket()
transcribing = self.Transcribing(self.audio_in_socket)
transcribing = self.TranscribingBehaviour(self.audio_in_socket)
transcribing.warmup()
self.add_behaviour(transcribing)

12
src/control_backend/agents/vad_agent.py → src/control_backend/agents/perception/vad_agent.py
View File

@@ -7,7 +7,7 @@ from spade.behaviour import CyclicBehaviour
from control_backend.agents import BaseAgent
from control_backend.core.config import settings
from .transcription.transcription_agent import TranscriptionAgent
from .transcription_agent.transcription_agent import TranscriptionAgent
class SocketPoller[T]:
@@ -44,7 +44,7 @@ class SocketPoller[T]:
return None
class Streaming(CyclicBehaviour):
class StreamingBehaviour(CyclicBehaviour):
def __init__(self, audio_in_socket: azmq.Socket, audio_out_socket: azmq.Socket):
super().__init__()
self.audio_in_poller = SocketPoller[bytes](audio_in_socket)
@@ -120,8 +120,8 @@ class VADAgent(BaseAgent):
"""
def __init__(self, audio_in_address: str, audio_in_bind: bool):
jid = settings.agent_settings.vad_agent_name + "@" + settings.agent_settings.host
super().__init__(jid, settings.agent_settings.vad_agent_name)
jid = settings.agent_settings.vad_name + "@" + settings.agent_settings.host
super().__init__(jid, settings.agent_settings.vad_name)
self.audio_in_address = audio_in_address
self.audio_in_bind = audio_in_bind
@@ -129,7 +129,7 @@ class VADAgent(BaseAgent):
self.audio_in_socket: azmq.Socket | None = None
self.audio_out_socket: azmq.Socket | None = None
self.streaming_behaviour: Streaming | None = None
self.streaming_behaviour: StreamingBehaviour | None = None
async def stop(self):
"""
@@ -173,7 +173,7 @@ class VADAgent(BaseAgent):
return
audio_out_address = f"tcp://localhost:{audio_out_port}"
self.streaming_behaviour = Streaming(self.audio_in_socket, self.audio_out_socket)
self.streaming_behaviour = StreamingBehaviour(self.audio_in_socket, self.audio_out_socket)
self.add_behaviour(self.streaming_behaviour)
# Start agents dependent on the output audio fragments here

162
src/control_backend/agents/ri_communication_agent.py
View File

@@ -1,162 +0,0 @@
import asyncio
import zmq
from spade.behaviour import CyclicBehaviour
from zmq.asyncio import Context
from control_backend.agents import BaseAgent
from control_backend.core.config import settings
from .ri_command_agent import RICommandAgent
class RICommunicationAgent(BaseAgent):
req_socket: zmq.Socket
_address = ""
_bind = True
def __init__(
self,
jid: str,
password: str,
port: int = settings.agent_settings.default_spade_port,
verify_security: bool = False,
address=settings.zmq_settings.ri_command_address,
bind=False,
):
super().__init__(jid, password, port, verify_security)
self._address = address
self._bind = bind
class ListenBehaviour(CyclicBehaviour):
async def run(self):
"""
Run the listening (ping) loop indefinetely.
"""
assert self.agent is not None
# We need to listen and sent pings.
message = {"endpoint": "ping", "data": {"id": "e.g. some reference id"}}
await self.agent.req_socket.send_json(message)
# Wait up to three seconds for a reply:)
try:
message = await asyncio.wait_for(self.agent.req_socket.recv_json(), timeout=3.0)
# We didnt get a reply :(
except TimeoutError:
self.agent.logger.info("No ping retrieved in 3 seconds, killing myself.")
self.kill()
self.agent.logger.debug('Received message "%s"', message)
if "endpoint" not in message:
self.agent.logger.error("No received endpoint in message, excepted ping endpoint.")
return
# See what endpoint we received
match message["endpoint"]:
case "ping":
await asyncio.sleep(settings.behaviour_settings.ping_sleep_s)
case _:
self.agent.logger.info(
"Received message with topic different than ping, while ping expected."
)
async def setup(self, max_retries: int = settings.behaviour_settings.comm_setup_max_retries):
"""
Try to setup the communication agent, we have 5 retries in case we dont have a response yet.
"""
self.logger.info("Setting up %s", self.jid)
retries = 0
# Let's try a certain amount of times before failing connection
while retries < max_retries:
# Bind request socket
self.req_socket = Context.instance().socket(zmq.REQ)
if self._bind:
self.req_socket.bind(self._address)
else:
self.req_socket.connect(self._address)
# Send our message and receive one back:)
message = {"endpoint": "negotiate/ports", "data": None}
await self.req_socket.send_json(message)
try:
received_message = await asyncio.wait_for(self.req_socket.recv_json(), timeout=20.0)
except TimeoutError:
self.logger.warning(
"No connection established in 20 seconds (attempt %d/%d)",
retries + 1,
max_retries,
)
retries += 1
continue
except Exception as e:
self.logger.error("Unexpected error during negotiation: %s", e)
retries += 1
continue
# Validate endpoint
endpoint = received_message.get("endpoint")
if endpoint != "negotiate/ports":
# TODO: Should this send a message back?
self.logger.error(
"Invalid endpoint '%s' received (attempt %d/%d)",
endpoint,
retries + 1,
max_retries,
)
retries += 1
continue
# At this point, we have a valid response
try:
for port_data in received_message["data"]:
id = port_data["id"]
port = port_data["port"]
bind = port_data["bind"]
if not bind:
addr = f"tcp://localhost:{port}"
else:
addr = f"tcp://*:{port}"
match id:
case "main":
if addr != self._address:
if not bind:
self.req_socket.connect(addr)
else:
self.req_socket.bind(addr)
case "actuation":
ri_commands_agent = RICommandAgent(
settings.agent_settings.ri_command_agent_name
+ "@"
+ settings.agent_settings.host,
settings.agent_settings.ri_command_agent_name,
address=addr,
bind=bind,
)
await ri_commands_agent.start()
case _:
self.logger.warning("Unhandled negotiation id: %s", id)
except Exception as e:
self.logger.error("Error unpacking negotiation data: %s", e)
retries += 1
continue
# setup succeeded
break
else:
self.logger.error("Failed to set up RICommunicationAgent after %d retries", max_retries)
return
# Set up ping behaviour
listen_behaviour = self.ListenBehaviour()
self.add_behaviour(listen_behaviour)
self.logger.info("Finished setting up %s", self.jid)

20
src/control_backend/api/v1/endpoints/command.py
View File

@@ -1,20 +0,0 @@
import logging
from fastapi import APIRouter, Request
from control_backend.schemas.ri_message import SpeechCommand
logger = logging.getLogger(__name__)
router = APIRouter()
@router.post("/command", status_code=202)
async def receive_command(command: SpeechCommand, request: Request):
# Validate and retrieve data.
SpeechCommand.model_validate(command)
topic = b"command"
pub_socket = request.app.state.endpoints_pub_socket
await pub_socket.send_multipart([topic, command.model_dump_json().encode()])
return {"status": "Command received"}

25
src/control_backend/api/v1/endpoints/program.py Normal file
View File

@@ -0,0 +1,25 @@
import logging
from fastapi import APIRouter, Request
from control_backend.schemas.program import Program
logger = logging.getLogger(__name__)
router = APIRouter()
@router.post("/program", status_code=202)
async def receive_message(program: Program, request: Request):
"""
Receives a BehaviorProgram, pydantic checks it.
Converts it into real Phase objects.
"""
logger.debug("Received raw program: %s", program)
# send away
topic = b"program"
body = program.model_dump_json().encode()
pub_socket = request.app.state.endpoints_pub_socket
await pub_socket.send_multipart([topic, body])
return {"status": "Program parsed"}

71
src/control_backend/api/v1/endpoints/robot.py Normal file
View File

@@ -0,0 +1,71 @@
import asyncio
import json
import logging
import zmq.asyncio
from fastapi import APIRouter, Request
from fastapi.responses import StreamingResponse
from zmq.asyncio import Context, Socket
from control_backend.core.config import settings
from control_backend.schemas.ri_message import SpeechCommand
logger = logging.getLogger(__name__)
router = APIRouter()
@router.post("/command", status_code=202)
async def receive_command(command: SpeechCommand, request: Request):
# Validate and retrieve data.
SpeechCommand.model_validate(command)
topic = b"command"
pub_socket: Socket = request.app.state.endpoints_pub_socket
await pub_socket.send_multipart([topic, command.model_dump_json().encode()])
return {"status": "Command received"}
@router.get("/ping_check")
async def ping(request: Request):
pass
@router.get("/ping_stream")
async def ping_stream(request: Request):
"""Stream live updates whenever the device state changes."""
async def event_stream():
# Set up internal socket to receive ping updates
sub_socket = Context.instance().socket(zmq.SUB)
sub_socket.connect(settings.zmq_settings.internal_sub_address)
sub_socket.setsockopt(zmq.SUBSCRIBE, b"ping")
connected = False
ping_frequency = 2
# Even though its most likely the updates should alternate
# (So, True - False - True - False for connectivity),
# let's still check.
while True:
try:
topic, body = await asyncio.wait_for(
sub_socket.recv_multipart(), timeout=ping_frequency
)
connected = json.loads(body)
except TimeoutError:
logger.debug("got timeout error in ping loop in ping router")
connected = False
# Stop if client disconnected
if await request.is_disconnected():
logger.info("Client disconnected from SSE")
break
logger.debug(f"Yielded new connection event in robot ping router: {str(connected)}")
connectedJson = json.dumps(connected)
yield (f"data: {connectedJson}\n\n")
return StreamingResponse(event_stream(), media_type="text/event-stream")

6
src/control_backend/api/v1/router.py
View File

@@ -1,6 +1,6 @@
from fastapi.routing import APIRouter
from control_backend.api.v1.endpoints import command, logs, message, sse
from control_backend.api.v1.endpoints import logs, message, program, robot, sse
api_router = APIRouter()
@@ -8,6 +8,8 @@ api_router.include_router(message.router, tags=["Messages"])
api_router.include_router(sse.router, tags=["SSE"])
api_router.include_router(command.router, tags=["Commands"])
api_router.include_router(robot.router, prefix="/robot", tags=["Pings", "Commands"])
api_router.include_router(logs.router, tags=["Logs"])
api_router.include_router(program.router, tags=["Program"])

20
src/control_backend/core/config.py
View File

@@ -15,22 +15,22 @@ class AgentSettings(BaseModel):
host: str = "localhost"
# agent names
bdi_core_agent_name: str = "bdi_core"
belief_collector_agent_name: str = "belief_collector"
text_belief_extractor_agent_name: str = "text_belief_extractor"
vad_agent_name: str = "vad_agent"
llm_agent_name: str = "llm_agent"
test_agent_name: str = "test_agent"
transcription_agent_name: str = "transcription_agent"
ri_communication_agent_name: str = "ri_communication_agent"
ri_command_agent_name: str = "ri_command_agent"
bdi_core_name: str = "bdi_core_agent"
bdi_belief_collector_name: str = "belief_collector_agent"
text_belief_extractor_name: str = "text_belief_extractor_agent"
vad_name: str = "vad_agent"
llm_name: str = "llm_agent"
test_name: str = "test_agent"
transcription_name: str = "transcription_agent"
ri_communication_name: str = "ri_communication_agent"
robot_speech_name: str = "robot_speech_agent"
# default SPADE port
default_spade_port: int = 5222
class BehaviourSettings(BaseModel):
ping_sleep_s: float = 1.0
sleep_s: float = 1.0
comm_setup_max_retries: int = 5
socket_poller_timeout_ms: int = 100

69
src/control_backend/main.py
View File

@@ -7,13 +7,25 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from zmq.asyncio import Context
from control_backend.agents import (
BeliefCollectorAgent,
LLMAgent,
RICommunicationAgent,
VADAgent,
# Act agents
# BDI agents
from control_backend.agents.bdi import (
BDIBeliefCollectorAgent,
BDICoreAgent,
TextBeliefExtractorAgent,
)
from control_backend.agents.bdi import BDICoreAgent, TBeliefExtractorAgent
# Communication agents
from control_backend.agents.communication import RICommunicationAgent
# Emotional Agents
# LLM Agents
from control_backend.agents.llm import LLMAgent
# Perceive agents
from control_backend.agents.perception import VADAgent
# Other backend imports
from control_backend.api.v1.router import api_router
from control_backend.core.config import settings
from control_backend.logging import setup_logging
@@ -67,10 +79,10 @@ async def lifespan(app: FastAPI):
"RICommunicationAgent": (
RICommunicationAgent,
{
"name": settings.agent_settings.ri_communication_agent_name,
"jid": f"{settings.agent_settings.ri_communication_agent_name}"
"name": settings.agent_settings.ri_communication_name,
"jid": f"{settings.agent_settings.ri_communication_name}"
f"@{settings.agent_settings.host}",
"password": settings.agent_settings.ri_communication_agent_name,
"password": settings.agent_settings.ri_communication_name,
"address": settings.zmq_settings.ri_communication_address,
"bind": True,
},
@@ -78,37 +90,36 @@ async def lifespan(app: FastAPI):
"LLMAgent": (
LLMAgent,
{
"name": settings.agent_settings.llm_agent_name,
"jid": f"{settings.agent_settings.llm_agent_name}@{settings.agent_settings.host}",
"password": settings.agent_settings.llm_agent_name,
"name": settings.agent_settings.llm_name,
"jid": f"{settings.agent_settings.llm_name}@{settings.agent_settings.host}",
"password": settings.agent_settings.llm_name,
},
),
"BDICoreAgent": (
BDICoreAgent,
{
"name": settings.agent_settings.bdi_core_agent_name,
"jid": f"{settings.agent_settings.bdi_core_agent_name}@"
f"{settings.agent_settings.host}",
"password": settings.agent_settings.bdi_core_agent_name,
"asl": "src/control_backend/agents/bdi/rules.asl",
"name": settings.agent_settings.bdi_core_name,
"jid": f"{settings.agent_settings.bdi_core_name}@{settings.agent_settings.host}",
"password": settings.agent_settings.bdi_core_name,
"asl": "src/control_backend/agents/bdi/bdi_core_agent/rules.asl",
},
),
"BeliefCollectorAgent": (
BeliefCollectorAgent,
BDIBeliefCollectorAgent,
{
"name": settings.agent_settings.belief_collector_agent_name,
"jid": f"{settings.agent_settings.belief_collector_agent_name}@"
"name": settings.agent_settings.bdi_belief_collector_name,
"jid": f"{settings.agent_settings.bdi_belief_collector_name}@"
f"{settings.agent_settings.host}",
"password": settings.agent_settings.belief_collector_agent_name,
"password": settings.agent_settings.bdi_belief_collector_name,
},
),
"TBeliefExtractor": (
TBeliefExtractorAgent,
"TextBeliefExtractorAgent": (
TextBeliefExtractorAgent,
{
"name": settings.agent_settings.text_belief_extractor_agent_name,
"jid": f"{settings.agent_settings.text_belief_extractor_agent_name}@"
"name": settings.agent_settings.text_belief_extractor_name,
"jid": f"{settings.agent_settings.text_belief_extractor_name}@"
f"{settings.agent_settings.host}",
"password": settings.agent_settings.text_belief_extractor_agent_name,
"password": settings.agent_settings.text_belief_extractor_name,
},
),
"VADAgent": (
@@ -117,17 +128,23 @@ async def lifespan(app: FastAPI):
),
}
vad_agent = None
for name, (agent_class, kwargs) in agents_to_start.items():
try:
logger.debug("Starting agent: %s", name)
agent_instance = agent_class(**{k: v for k, v in kwargs.items() if k != "name"})
await agent_instance.start()
if isinstance(agent_instance, VADAgent):
vad_agent = agent_instance
logger.info("Agent '%s' started successfully.", name)
except Exception as e:
logger.error("Failed to start agent '%s': %s", name, e, exc_info=True)
# Consider if the application should continue if an agent fails to start.
raise
await vad_agent.streaming_behaviour.reset()
logger.info("Application startup complete.")
yield

38
src/control_backend/schemas/program.py Normal file
View File

@@ -0,0 +1,38 @@
from pydantic import BaseModel
class Norm(BaseModel):
id: str
name: str
value: str
class Goal(BaseModel):
id: str
name: str
description: str
achieved: bool
class Trigger(BaseModel):
id: str
label: str
type: str
value: list[str]
class PhaseData(BaseModel):
norms: list[Norm]
goals: list[Goal]
triggers: list[Trigger]
class Phase(BaseModel):
id: str
name: str
nextPhaseId: str
phaseData: PhaseData
class Program(BaseModel):
phases: list[Phase]