Merge remote-tracking branch 'origin/dev' into refactor/config-file
# Conflicts: # src/control_backend/agents/ri_communication_agent.py # src/control_backend/core/config.py # src/control_backend/main.py
This commit is contained in:
@@ -1,7 +1 @@
|
||||
from .base import BaseAgent as BaseAgent
|
||||
from .belief_collector.belief_collector import BeliefCollectorAgent as BeliefCollectorAgent
|
||||
from .llm.llm import LLMAgent as LLMAgent
|
||||
from .ri_command_agent import RICommandAgent as RICommandAgent
|
||||
from .ri_communication_agent import RICommunicationAgent as RICommunicationAgent
|
||||
from .transcription.transcription_agent import TranscriptionAgent as TranscriptionAgent
|
||||
from .vad_agent import VADAgent as VADAgent
|
||||
|
||||
1
src/control_backend/agents/actuation/__init__.py
Normal file
1
src/control_backend/agents/actuation/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
from .robot_speech_agent import RobotSpeechAgent as RobotSpeechAgent
|
||||
@@ -10,7 +10,7 @@ from control_backend.core.config import settings
|
||||
from control_backend.schemas.ri_message import SpeechCommand
|
||||
|
||||
|
||||
class RICommandAgent(BaseAgent):
|
||||
class RobotSpeechAgent(BaseAgent):
|
||||
subsocket: zmq.Socket
|
||||
pubsocket: zmq.Socket
|
||||
address = ""
|
||||
@@ -29,7 +29,7 @@ class RICommandAgent(BaseAgent):
|
||||
self.address = address
|
||||
self.bind = bind
|
||||
|
||||
class SendCommandsBehaviour(CyclicBehaviour):
|
||||
class SendZMQCommandsBehaviour(CyclicBehaviour):
|
||||
"""Behaviour for sending commands received from the UI."""
|
||||
|
||||
async def run(self):
|
||||
@@ -50,7 +50,7 @@ class RICommandAgent(BaseAgent):
|
||||
except Exception as e:
|
||||
self.agent.logger.error("Error processing message: %s", e)
|
||||
|
||||
class SendPythonCommandsBehaviour(CyclicBehaviour):
|
||||
class SendSpadeCommandsBehaviour(CyclicBehaviour):
|
||||
"""Behaviour for sending commands received from other Python agents."""
|
||||
|
||||
async def run(self):
|
||||
@@ -64,7 +64,7 @@ class RICommandAgent(BaseAgent):
|
||||
|
||||
async def setup(self):
|
||||
"""
|
||||
Setup the command agent
|
||||
Setup the robot speech command agent
|
||||
"""
|
||||
self.logger.info("Setting up %s", self.jid)
|
||||
|
||||
@@ -72,7 +72,7 @@ class RICommandAgent(BaseAgent):
|
||||
|
||||
# To the robot
|
||||
self.pubsocket = context.socket(zmq.PUB)
|
||||
if self.bind:
|
||||
if self.bind: # TODO: Should this ever be the case?
|
||||
self.pubsocket.bind(self.address)
|
||||
else:
|
||||
self.pubsocket.connect(self.address)
|
||||
@@ -83,8 +83,8 @@ class RICommandAgent(BaseAgent):
|
||||
self.subsocket.setsockopt(zmq.SUBSCRIBE, b"command")
|
||||
|
||||
# Add behaviour to our agent
|
||||
commands_behaviour = self.SendCommandsBehaviour()
|
||||
commands_behaviour = self.SendZMQCommandsBehaviour()
|
||||
self.add_behaviour(commands_behaviour)
|
||||
self.add_behaviour(self.SendPythonCommandsBehaviour())
|
||||
self.add_behaviour(self.SendSpadeCommandsBehaviour())
|
||||
|
||||
self.logger.info("Finished setting up %s", self.jid)
|
||||
@@ -1,2 +1,7 @@
|
||||
from .bdi_core import BDICoreAgent as BDICoreAgent
|
||||
from .text_extractor import TBeliefExtractorAgent as TBeliefExtractorAgent
|
||||
from .bdi_core_agent.bdi_core_agent import BDICoreAgent as BDICoreAgent
|
||||
from .belief_collector_agent.belief_collector_agent import (
|
||||
BDIBeliefCollectorAgent as BDIBeliefCollectorAgent,
|
||||
)
|
||||
from .text_belief_extractor_agent.text_belief_extractor_agent import (
|
||||
TextBeliefExtractorAgent as TextBeliefExtractorAgent,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ from spade_bdi.bdi import BDIAgent
|
||||
|
||||
from control_backend.core.config import settings
|
||||
|
||||
from .behaviours.belief_setter import BeliefSetterBehaviour
|
||||
from .behaviours.belief_setter_behaviour import BeliefSetterBehaviour
|
||||
from .behaviours.receive_llm_resp_behaviour import ReceiveLLMResponseBehaviour
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ class BDICoreAgent(BDIAgent):
|
||||
class SendBehaviour(OneShotBehaviour):
|
||||
async def run(self) -> None:
|
||||
msg = Message(
|
||||
to=settings.agent_settings.llm_agent_name + "@" + settings.agent_settings.host,
|
||||
to=settings.agent_settings.llm_name + "@" + settings.agent_settings.host,
|
||||
body=text,
|
||||
)
|
||||
|
||||
@@ -32,7 +32,7 @@ class BeliefSetterBehaviour(CyclicBehaviour):
|
||||
self.agent.logger.debug("Processing message from sender: %s", sender)
|
||||
|
||||
match sender:
|
||||
case settings.agent_settings.belief_collector_agent_name:
|
||||
case settings.agent_settings.bdi_belief_collector_name:
|
||||
self.agent.logger.debug(
|
||||
"Message is from the belief collector agent. Processing as belief message."
|
||||
)
|
||||
@@ -15,14 +15,14 @@ class ReceiveLLMResponseBehaviour(CyclicBehaviour):
|
||||
|
||||
sender = msg.sender.node
|
||||
match sender:
|
||||
case settings.agent_settings.llm_agent_name:
|
||||
case settings.agent_settings.llm_name:
|
||||
content = msg.body
|
||||
self.agent.logger.info("Received LLM response: %s", content)
|
||||
|
||||
speech_command = SpeechCommand(data=content)
|
||||
|
||||
message = Message(
|
||||
to=settings.agent_settings.ri_command_agent_name
|
||||
to=settings.agent_settings.robot_speech_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host,
|
||||
sender=self.agent.jid,
|
||||
@@ -7,7 +7,7 @@ from spade.behaviour import CyclicBehaviour
|
||||
from control_backend.core.config import settings
|
||||
|
||||
|
||||
class ContinuousBeliefCollector(CyclicBehaviour):
|
||||
class BeliefCollectorBehaviour(CyclicBehaviour):
|
||||
"""
|
||||
Continuously collects beliefs/emotions from extractor agents:
|
||||
Then we send a unified belief packet to the BDI agent.
|
||||
@@ -35,7 +35,7 @@ class ContinuousBeliefCollector(CyclicBehaviour):
|
||||
msg_type = payload.get("type")
|
||||
|
||||
# Prefer explicit 'type' field
|
||||
if msg_type == "belief_extraction_text" or sender_node == "belief_text_agent_mock":
|
||||
if msg_type == "belief_extraction_text" or sender_node == "bel_text_agent_mock":
|
||||
self.agent.logger.debug(
|
||||
"Message routed to _handle_belief_text (sender=%s)", sender_node
|
||||
)
|
||||
@@ -83,7 +83,7 @@ class ContinuousBeliefCollector(CyclicBehaviour):
|
||||
if not beliefs:
|
||||
return
|
||||
|
||||
to_jid = f"{settings.agent_settings.bdi_core_agent_name}@{settings.agent_settings.host}"
|
||||
to_jid = f"{settings.agent_settings.bdi_core_name}@{settings.agent_settings.host}"
|
||||
|
||||
msg = Message(to=to_jid, sender=self.agent.jid, thread="beliefs")
|
||||
msg.body = json.dumps(beliefs)
|
||||
@@ -0,0 +1,11 @@
|
||||
from control_backend.agents.base import BaseAgent
|
||||
|
||||
from .behaviours.belief_collector_behaviour import BeliefCollectorBehaviour
|
||||
|
||||
|
||||
class BDIBeliefCollectorAgent(BaseAgent):
|
||||
async def setup(self):
|
||||
self.logger.info("BDIBeliefCollectorAgent starting (%s)", self.jid)
|
||||
# Attach the continuous collector behaviour (listens and forwards to BDI)
|
||||
self.add_behaviour(BeliefCollectorBehaviour())
|
||||
self.logger.info("BDIBeliefCollectorAgent ready.")
|
||||
@@ -7,7 +7,7 @@ from spade.message import Message
|
||||
from control_backend.core.config import settings
|
||||
|
||||
|
||||
class BeliefFromText(CyclicBehaviour):
|
||||
class TextBeliefExtractorBehaviour(CyclicBehaviour):
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# TODO: LLM prompt nog hardcoded
|
||||
@@ -44,7 +44,7 @@ class BeliefFromText(CyclicBehaviour):
|
||||
|
||||
sender = msg.sender.node
|
||||
match sender:
|
||||
case settings.agent_settings.transcription_agent_name:
|
||||
case settings.agent_settings.transcription_name:
|
||||
self.logger.debug("Received text from transcriber: %s", msg.body)
|
||||
await self._process_transcription_demo(msg.body)
|
||||
case _:
|
||||
@@ -71,7 +71,7 @@ class BeliefFromText(CyclicBehaviour):
|
||||
belief_message = Message()
|
||||
|
||||
belief_message.to = (
|
||||
settings.agent_settings.belief_collector_agent_name
|
||||
settings.agent_settings.bdi_belief_collector_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host
|
||||
)
|
||||
@@ -95,7 +95,7 @@ class BeliefFromText(CyclicBehaviour):
|
||||
belief_msg = Message()
|
||||
|
||||
belief_msg.to = (
|
||||
settings.agent_settings.belief_collector_agent_name + "@" + settings.agent_settings.host
|
||||
settings.agent_settings.bdi_belief_collector_name + "@" + settings.agent_settings.host
|
||||
)
|
||||
belief_msg.body = payload
|
||||
belief_msg.thread = "beliefs"
|
||||
@@ -0,0 +1,8 @@
|
||||
from control_backend.agents.base import BaseAgent
|
||||
|
||||
from .behaviours.text_belief_extractor_behaviour import TextBeliefExtractorBehaviour
|
||||
|
||||
|
||||
class TextBeliefExtractorAgent(BaseAgent):
|
||||
async def setup(self):
|
||||
self.add_behaviour(TextBeliefExtractorBehaviour())
|
||||
@@ -1,8 +0,0 @@
|
||||
from control_backend.agents.base import BaseAgent
|
||||
|
||||
from .behaviours.text_belief_extractor import BeliefFromText
|
||||
|
||||
|
||||
class TBeliefExtractorAgent(BaseAgent):
|
||||
async def setup(self):
|
||||
self.add_behaviour(BeliefFromText())
|
||||
@@ -1,11 +0,0 @@
|
||||
from control_backend.agents.base import BaseAgent
|
||||
|
||||
from .behaviours.continuous_collect import ContinuousBeliefCollector
|
||||
|
||||
|
||||
class BeliefCollectorAgent(BaseAgent):
|
||||
async def setup(self):
|
||||
self.logger.info("BeliefCollectorAgent starting (%s)", self.jid)
|
||||
# Attach the continuous collector behaviour (listens and forwards to BDI)
|
||||
self.add_behaviour(ContinuousBeliefCollector())
|
||||
self.logger.info("BeliefCollectorAgent ready.")
|
||||
1
src/control_backend/agents/communication/__init__.py
Normal file
1
src/control_backend/agents/communication/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
from .ri_communication_agent import RICommunicationAgent as RICommunicationAgent
|
||||
@@ -0,0 +1,250 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import zmq.asyncio
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
from zmq.asyncio import Context
|
||||
|
||||
from control_backend.agents import BaseAgent
|
||||
from control_backend.core.config import settings
|
||||
|
||||
from ..actuation.robot_speech_agent import RobotSpeechAgent
|
||||
|
||||
|
||||
class RICommunicationAgent(BaseAgent):
|
||||
req_socket: zmq.Socket
|
||||
_address = ""
|
||||
_bind = True
|
||||
connected = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
jid: str,
|
||||
password: str,
|
||||
port: int = settings.agent_settings.default_spade_port,
|
||||
verify_security: bool = False,
|
||||
address=settings.zmq_settings.ri_command_address,
|
||||
bind=False,
|
||||
):
|
||||
super().__init__(jid, password, port, verify_security)
|
||||
self._address = address
|
||||
self._bind = bind
|
||||
self._req_socket: zmq.asyncio.Socket | None = None
|
||||
self.pub_socket: zmq.asyncio.Socket | None = None
|
||||
|
||||
class ListenBehaviour(CyclicBehaviour):
|
||||
async def run(self):
|
||||
"""
|
||||
Run the listening (ping) loop indefinetely.
|
||||
"""
|
||||
assert self.agent is not None
|
||||
|
||||
if not self.agent.connected:
|
||||
await asyncio.sleep(settings.behaviour_settings.sleep_s)
|
||||
return
|
||||
|
||||
# We need to listen and sent pings.
|
||||
message = {"endpoint": "ping", "data": {"id": "e.g. some reference id"}}
|
||||
seconds_to_wait_total = settings.behaviour_settings.sleep_s
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self.agent._req_socket.send_json(message), timeout=seconds_to_wait_total / 2
|
||||
)
|
||||
except TimeoutError:
|
||||
self.agent.logger.debug(
|
||||
"Waited too long to send message - "
|
||||
"we probably dont have any receivers... but let's check!"
|
||||
)
|
||||
|
||||
# Wait up to {seconds_to_wait_total/2} seconds for a reply
|
||||
try:
|
||||
message = await asyncio.wait_for(
|
||||
self.agent._req_socket.recv_json(), timeout=seconds_to_wait_total / 2
|
||||
)
|
||||
|
||||
# We didnt get a reply
|
||||
except TimeoutError:
|
||||
self.agent.logger.info(
|
||||
f"No ping retrieved in {seconds_to_wait_total} seconds, "
|
||||
"sending UI disconnection event and attempting to restart."
|
||||
)
|
||||
|
||||
# Make sure we dont retry receiving messages untill we're setup.
|
||||
self.agent.connected = False
|
||||
self.agent.remove_behaviour(self)
|
||||
|
||||
# Tell UI we're disconnected.
|
||||
topic = b"ping"
|
||||
data = json.dumps(False).encode()
|
||||
if self.agent.pub_socket is None:
|
||||
self.agent.logger.warning(
|
||||
"Communication agent pub socket not correctly initialized."
|
||||
)
|
||||
else:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self.agent.pub_socket.send_multipart([topic, data]), 5
|
||||
)
|
||||
except TimeoutError:
|
||||
self.agent.logger.warning(
|
||||
f"Initial connection ping for router timed out in {self.agent.name}."
|
||||
)
|
||||
|
||||
# Try to reboot.
|
||||
self.agent.logger.debug("Restarting communication agent.")
|
||||
await self.agent.setup()
|
||||
|
||||
self.agent.logger.debug(f'Received message "{message}" from RI.')
|
||||
if "endpoint" not in message:
|
||||
self.agent.logger.warning(
|
||||
"No received endpoint in message, expected ping endpoint."
|
||||
)
|
||||
return
|
||||
|
||||
# See what endpoint we received
|
||||
match message["endpoint"]:
|
||||
case "ping":
|
||||
topic = b"ping"
|
||||
data = json.dumps(True).encode()
|
||||
if self.agent.pub_socket is not None:
|
||||
await self.agent.pub_socket.send_multipart([topic, data])
|
||||
await asyncio.sleep(settings.behaviour_settings.sleep_s)
|
||||
case _:
|
||||
self.agent.logger.debug(
|
||||
"Received message with topic different than ping, while ping expected."
|
||||
)
|
||||
|
||||
async def setup_sockets(self, force=False):
|
||||
"""
|
||||
Sets up request socket for communication agent.
|
||||
"""
|
||||
# Bind request socket
|
||||
if self._req_socket is None or force:
|
||||
self._req_socket = Context.instance().socket(zmq.REQ)
|
||||
if self._bind:
|
||||
self._req_socket.bind(self._address)
|
||||
else:
|
||||
self._req_socket.connect(self._address)
|
||||
|
||||
if self.pub_socket is None or force:
|
||||
self.pub_socket = Context.instance().socket(zmq.PUB)
|
||||
self.pub_socket.connect(settings.zmq_settings.internal_pub_address)
|
||||
|
||||
async def setup(self, max_retries: int = settings.behaviour_settings.comm_setup_max_retries):
|
||||
"""
|
||||
Try to set up the communication agent, we have `behaviour_settings.comm_setup_max_retries`
|
||||
retries in case we don't have a response yet.
|
||||
"""
|
||||
self.logger.info("Setting up %s", self.jid)
|
||||
|
||||
# Bind request socket
|
||||
await self.setup_sockets()
|
||||
|
||||
retries = 0
|
||||
# Let's try a certain amount of times before failing connection
|
||||
while retries < max_retries:
|
||||
# Make sure the socket is properly setup.
|
||||
if self._req_socket is None:
|
||||
continue
|
||||
|
||||
# Send our message and receive one back
|
||||
message = {"endpoint": "negotiate/ports", "data": {}}
|
||||
await self._req_socket.send_json(message)
|
||||
|
||||
retry_frequency = 1.0
|
||||
try:
|
||||
received_message = await asyncio.wait_for(
|
||||
self._req_socket.recv_json(), timeout=retry_frequency
|
||||
)
|
||||
|
||||
except TimeoutError:
|
||||
self.logger.warning(
|
||||
"No connection established in %d seconds (attempt %d/%d)",
|
||||
retries * retry_frequency,
|
||||
retries + 1,
|
||||
max_retries,
|
||||
)
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning("Unexpected error during negotiation: %s", e)
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
# Validate endpoint
|
||||
endpoint = received_message.get("endpoint")
|
||||
if endpoint != "negotiate/ports":
|
||||
self.logger.warning(
|
||||
"Invalid endpoint '%s' received (attempt %d/%d)",
|
||||
endpoint,
|
||||
retries + 1,
|
||||
max_retries,
|
||||
)
|
||||
retries += 1
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
|
||||
# At this point, we have a valid response
|
||||
try:
|
||||
for port_data in received_message["data"]:
|
||||
id = port_data["id"]
|
||||
port = port_data["port"]
|
||||
bind = port_data["bind"]
|
||||
|
||||
if not bind:
|
||||
addr = f"tcp://localhost:{port}"
|
||||
else:
|
||||
addr = f"tcp://*:{port}"
|
||||
|
||||
match id:
|
||||
case "main":
|
||||
if addr != self._address:
|
||||
if not bind:
|
||||
self._req_socket.connect(addr)
|
||||
else:
|
||||
self._req_socket.bind(addr)
|
||||
case "actuation":
|
||||
ri_commands_agent = RobotSpeechAgent(
|
||||
settings.agent_settings.robot_speech_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host,
|
||||
settings.agent_settings.robot_speech_name,
|
||||
address=addr,
|
||||
bind=bind,
|
||||
)
|
||||
await ri_commands_agent.start()
|
||||
case _:
|
||||
self.logger.warning("Unhandled negotiation id: %s", id)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning("Error unpacking negotiation data: %s", e)
|
||||
retries += 1
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
|
||||
# setup succeeded
|
||||
break
|
||||
|
||||
else:
|
||||
self.logger.warning("Failed to set up %s after %d retries", self.name, max_retries)
|
||||
return
|
||||
|
||||
# Set up ping behaviour
|
||||
listen_behaviour = self.ListenBehaviour()
|
||||
self.add_behaviour(listen_behaviour)
|
||||
|
||||
# Let UI know that we're connected
|
||||
topic = b"ping"
|
||||
data = json.dumps(True).encode()
|
||||
if self.pub_socket is None:
|
||||
self.logger.warning("Communication agent pub socket not correctly initialized.")
|
||||
else:
|
||||
try:
|
||||
await asyncio.wait_for(self.pub_socket.send_multipart([topic, data]), 5)
|
||||
except TimeoutError:
|
||||
self.logger.warning("Initial connection ping for router timed out in com_ri_agent.")
|
||||
|
||||
# Make sure to start listening now that we're connected.
|
||||
self.connected = True
|
||||
self.logger.info("Finished setting up %s", self.jid)
|
||||
1
src/control_backend/agents/llm/__init__.py
Normal file
1
src/control_backend/agents/llm/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
from .llm_agent import LLMAgent as LLMAgent
|
||||
@@ -39,7 +39,7 @@ class LLMAgent(BaseAgent):
|
||||
sender,
|
||||
)
|
||||
|
||||
if sender == settings.agent_settings.bdi_core_agent_name:
|
||||
if sender == settings.agent_settings.bdi_core_name:
|
||||
self.agent.logger.debug("Processing message from BDI Core Agent")
|
||||
await self._process_bdi_message(msg)
|
||||
else:
|
||||
@@ -63,7 +63,7 @@ class LLMAgent(BaseAgent):
|
||||
Sends a response message back to the BDI Core Agent.
|
||||
"""
|
||||
reply = Message(
|
||||
to=settings.agent_settings.bdi_core_agent_name + "@" + settings.agent_settings.host,
|
||||
to=settings.agent_settings.bdi_core_name + "@" + settings.agent_settings.host,
|
||||
body=msg,
|
||||
)
|
||||
await self.send(reply)
|
||||
4
src/control_backend/agents/perception/__init__.py
Normal file
4
src/control_backend/agents/perception/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
from .transcription_agent.transcription_agent import (
|
||||
TranscriptionAgent as TranscriptionAgent,
|
||||
)
|
||||
from .vad_agent import VADAgent as VADAgent
|
||||
@@ -19,13 +19,13 @@ class TranscriptionAgent(BaseAgent):
|
||||
"""
|
||||
|
||||
def __init__(self, audio_in_address: str):
|
||||
jid = settings.agent_settings.transcription_agent_name + "@" + settings.agent_settings.host
|
||||
super().__init__(jid, settings.agent_settings.transcription_agent_name)
|
||||
jid = settings.agent_settings.transcription_name + "@" + settings.agent_settings.host
|
||||
super().__init__(jid, settings.agent_settings.transcription_name)
|
||||
|
||||
self.audio_in_address = audio_in_address
|
||||
self.audio_in_socket: azmq.Socket | None = None
|
||||
|
||||
class Transcribing(CyclicBehaviour):
|
||||
class TranscribingBehaviour(CyclicBehaviour):
|
||||
def __init__(self, audio_in_socket: azmq.Socket):
|
||||
super().__init__()
|
||||
max_concurrent_tasks = settings.behaviour_settings.transcription_max_concurrent_tasks
|
||||
@@ -44,7 +44,7 @@ class TranscriptionAgent(BaseAgent):
|
||||
async def _share_transcription(self, transcription: str):
|
||||
"""Share a transcription to the other agents that depend on it."""
|
||||
receiver_jids = [
|
||||
settings.agent_settings.text_belief_extractor_agent_name
|
||||
settings.agent_settings.text_belief_extractor_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host,
|
||||
] # Set message receivers here
|
||||
@@ -80,7 +80,7 @@ class TranscriptionAgent(BaseAgent):
|
||||
|
||||
self._connect_audio_in_socket()
|
||||
|
||||
transcribing = self.Transcribing(self.audio_in_socket)
|
||||
transcribing = self.TranscribingBehaviour(self.audio_in_socket)
|
||||
transcribing.warmup()
|
||||
self.add_behaviour(transcribing)
|
||||
|
||||
@@ -7,7 +7,7 @@ from spade.behaviour import CyclicBehaviour
|
||||
from control_backend.agents import BaseAgent
|
||||
from control_backend.core.config import settings
|
||||
|
||||
from .transcription.transcription_agent import TranscriptionAgent
|
||||
from .transcription_agent.transcription_agent import TranscriptionAgent
|
||||
|
||||
|
||||
class SocketPoller[T]:
|
||||
@@ -44,7 +44,7 @@ class SocketPoller[T]:
|
||||
return None
|
||||
|
||||
|
||||
class Streaming(CyclicBehaviour):
|
||||
class StreamingBehaviour(CyclicBehaviour):
|
||||
def __init__(self, audio_in_socket: azmq.Socket, audio_out_socket: azmq.Socket):
|
||||
super().__init__()
|
||||
self.audio_in_poller = SocketPoller[bytes](audio_in_socket)
|
||||
@@ -120,8 +120,8 @@ class VADAgent(BaseAgent):
|
||||
"""
|
||||
|
||||
def __init__(self, audio_in_address: str, audio_in_bind: bool):
|
||||
jid = settings.agent_settings.vad_agent_name + "@" + settings.agent_settings.host
|
||||
super().__init__(jid, settings.agent_settings.vad_agent_name)
|
||||
jid = settings.agent_settings.vad_name + "@" + settings.agent_settings.host
|
||||
super().__init__(jid, settings.agent_settings.vad_name)
|
||||
|
||||
self.audio_in_address = audio_in_address
|
||||
self.audio_in_bind = audio_in_bind
|
||||
@@ -129,7 +129,7 @@ class VADAgent(BaseAgent):
|
||||
self.audio_in_socket: azmq.Socket | None = None
|
||||
self.audio_out_socket: azmq.Socket | None = None
|
||||
|
||||
self.streaming_behaviour: Streaming | None = None
|
||||
self.streaming_behaviour: StreamingBehaviour | None = None
|
||||
|
||||
async def stop(self):
|
||||
"""
|
||||
@@ -173,7 +173,7 @@ class VADAgent(BaseAgent):
|
||||
return
|
||||
audio_out_address = f"tcp://localhost:{audio_out_port}"
|
||||
|
||||
self.streaming_behaviour = Streaming(self.audio_in_socket, self.audio_out_socket)
|
||||
self.streaming_behaviour = StreamingBehaviour(self.audio_in_socket, self.audio_out_socket)
|
||||
self.add_behaviour(self.streaming_behaviour)
|
||||
|
||||
# Start agents dependent on the output audio fragments here
|
||||
@@ -1,162 +0,0 @@
|
||||
import asyncio
|
||||
|
||||
import zmq
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
from zmq.asyncio import Context
|
||||
|
||||
from control_backend.agents import BaseAgent
|
||||
from control_backend.core.config import settings
|
||||
|
||||
from .ri_command_agent import RICommandAgent
|
||||
|
||||
|
||||
class RICommunicationAgent(BaseAgent):
|
||||
req_socket: zmq.Socket
|
||||
_address = ""
|
||||
_bind = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
jid: str,
|
||||
password: str,
|
||||
port: int = settings.agent_settings.default_spade_port,
|
||||
verify_security: bool = False,
|
||||
address=settings.zmq_settings.ri_command_address,
|
||||
bind=False,
|
||||
):
|
||||
super().__init__(jid, password, port, verify_security)
|
||||
self._address = address
|
||||
self._bind = bind
|
||||
|
||||
class ListenBehaviour(CyclicBehaviour):
|
||||
async def run(self):
|
||||
"""
|
||||
Run the listening (ping) loop indefinetely.
|
||||
"""
|
||||
assert self.agent is not None
|
||||
|
||||
# We need to listen and sent pings.
|
||||
message = {"endpoint": "ping", "data": {"id": "e.g. some reference id"}}
|
||||
await self.agent.req_socket.send_json(message)
|
||||
|
||||
# Wait up to three seconds for a reply:)
|
||||
try:
|
||||
message = await asyncio.wait_for(self.agent.req_socket.recv_json(), timeout=3.0)
|
||||
|
||||
# We didnt get a reply :(
|
||||
except TimeoutError:
|
||||
self.agent.logger.info("No ping retrieved in 3 seconds, killing myself.")
|
||||
self.kill()
|
||||
|
||||
self.agent.logger.debug('Received message "%s"', message)
|
||||
if "endpoint" not in message:
|
||||
self.agent.logger.error("No received endpoint in message, excepted ping endpoint.")
|
||||
return
|
||||
|
||||
# See what endpoint we received
|
||||
match message["endpoint"]:
|
||||
case "ping":
|
||||
await asyncio.sleep(settings.behaviour_settings.ping_sleep_s)
|
||||
case _:
|
||||
self.agent.logger.info(
|
||||
"Received message with topic different than ping, while ping expected."
|
||||
)
|
||||
|
||||
async def setup(self, max_retries: int = settings.behaviour_settings.comm_setup_max_retries):
|
||||
"""
|
||||
Try to setup the communication agent, we have 5 retries in case we dont have a response yet.
|
||||
"""
|
||||
self.logger.info("Setting up %s", self.jid)
|
||||
retries = 0
|
||||
|
||||
# Let's try a certain amount of times before failing connection
|
||||
while retries < max_retries:
|
||||
# Bind request socket
|
||||
self.req_socket = Context.instance().socket(zmq.REQ)
|
||||
if self._bind:
|
||||
self.req_socket.bind(self._address)
|
||||
else:
|
||||
self.req_socket.connect(self._address)
|
||||
|
||||
# Send our message and receive one back:)
|
||||
message = {"endpoint": "negotiate/ports", "data": None}
|
||||
await self.req_socket.send_json(message)
|
||||
|
||||
try:
|
||||
received_message = await asyncio.wait_for(self.req_socket.recv_json(), timeout=20.0)
|
||||
|
||||
except TimeoutError:
|
||||
self.logger.warning(
|
||||
"No connection established in 20 seconds (attempt %d/%d)",
|
||||
retries + 1,
|
||||
max_retries,
|
||||
)
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("Unexpected error during negotiation: %s", e)
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
# Validate endpoint
|
||||
endpoint = received_message.get("endpoint")
|
||||
if endpoint != "negotiate/ports":
|
||||
# TODO: Should this send a message back?
|
||||
self.logger.error(
|
||||
"Invalid endpoint '%s' received (attempt %d/%d)",
|
||||
endpoint,
|
||||
retries + 1,
|
||||
max_retries,
|
||||
)
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
# At this point, we have a valid response
|
||||
try:
|
||||
for port_data in received_message["data"]:
|
||||
id = port_data["id"]
|
||||
port = port_data["port"]
|
||||
bind = port_data["bind"]
|
||||
|
||||
if not bind:
|
||||
addr = f"tcp://localhost:{port}"
|
||||
else:
|
||||
addr = f"tcp://*:{port}"
|
||||
|
||||
match id:
|
||||
case "main":
|
||||
if addr != self._address:
|
||||
if not bind:
|
||||
self.req_socket.connect(addr)
|
||||
else:
|
||||
self.req_socket.bind(addr)
|
||||
case "actuation":
|
||||
ri_commands_agent = RICommandAgent(
|
||||
settings.agent_settings.ri_command_agent_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host,
|
||||
settings.agent_settings.ri_command_agent_name,
|
||||
address=addr,
|
||||
bind=bind,
|
||||
)
|
||||
await ri_commands_agent.start()
|
||||
case _:
|
||||
self.logger.warning("Unhandled negotiation id: %s", id)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("Error unpacking negotiation data: %s", e)
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
# setup succeeded
|
||||
break
|
||||
|
||||
else:
|
||||
self.logger.error("Failed to set up RICommunicationAgent after %d retries", max_retries)
|
||||
return
|
||||
|
||||
# Set up ping behaviour
|
||||
listen_behaviour = self.ListenBehaviour()
|
||||
self.add_behaviour(listen_behaviour)
|
||||
self.logger.info("Finished setting up %s", self.jid)
|
||||
Reference in New Issue
Block a user