refactor: remove SPADE dependencies
Did not look at tests yet, this is a very non-final commit. ref: N25B-300
This commit is contained in:
@@ -1,11 +1,10 @@
|
||||
import json
|
||||
|
||||
import spade.agent
|
||||
import zmq
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
from zmq.asyncio import Context
|
||||
import zmq.asyncio as azmq
|
||||
|
||||
from control_backend.agents import BaseAgent
|
||||
from control_backend.core.agent_system import InternalMessage
|
||||
from control_backend.core.config import settings
|
||||
from control_backend.schemas.ri_message import SpeechCommand
|
||||
|
||||
@@ -18,57 +17,21 @@ class RobotSpeechAgent(BaseAgent):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
jid: str,
|
||||
password: str,
|
||||
port: int = settings.agent_settings.default_spade_port,
|
||||
verify_security: bool = False,
|
||||
name: str,
|
||||
address=settings.zmq_settings.ri_command_address,
|
||||
bind=False,
|
||||
):
|
||||
super().__init__(jid, password, port, verify_security)
|
||||
super().__init__(name)
|
||||
self.address = address
|
||||
self.bind = bind
|
||||
|
||||
class SendZMQCommandsBehaviour(CyclicBehaviour):
|
||||
"""Behaviour for sending commands received from the UI."""
|
||||
|
||||
async def run(self):
|
||||
"""
|
||||
Run the command publishing loop indefinetely.
|
||||
"""
|
||||
assert self.agent is not None
|
||||
# Get a message internally (with topic command)
|
||||
topic, body = await self.agent.subsocket.recv_multipart()
|
||||
|
||||
# Try to get body
|
||||
try:
|
||||
body = json.loads(body)
|
||||
message = SpeechCommand.model_validate(body)
|
||||
|
||||
# Send to the robot.
|
||||
await self.agent.pubsocket.send_json(message.model_dump())
|
||||
except Exception as e:
|
||||
self.agent.logger.error("Error processing message: %s", e)
|
||||
|
||||
class SendSpadeCommandsBehaviour(CyclicBehaviour):
|
||||
"""Behaviour for sending commands received from other Python agents."""
|
||||
|
||||
async def run(self):
|
||||
message: spade.agent.Message = await self.receive(timeout=0.1)
|
||||
if message and message.to == self.agent.jid:
|
||||
try:
|
||||
speech_command = SpeechCommand.model_validate_json(message.body)
|
||||
await self.agent.pubsocket.send_json(speech_command.model_dump())
|
||||
except Exception as e:
|
||||
self.agent.logger.error("Error processing message: %s", e)
|
||||
|
||||
async def setup(self):
|
||||
"""
|
||||
Setup the robot speech command agent
|
||||
"""
|
||||
self.logger.info("Setting up %s", self.jid)
|
||||
self.logger.info("Setting up %s", self.name)
|
||||
|
||||
context = Context.instance()
|
||||
context = azmq.Context.instance()
|
||||
|
||||
# To the robot
|
||||
self.pubsocket = context.socket(zmq.PUB)
|
||||
@@ -82,9 +45,38 @@ class RobotSpeechAgent(BaseAgent):
|
||||
self.subsocket.connect(settings.zmq_settings.internal_sub_address)
|
||||
self.subsocket.setsockopt(zmq.SUBSCRIBE, b"command")
|
||||
|
||||
# Add behaviour to our agent
|
||||
commands_behaviour = self.SendZMQCommandsBehaviour()
|
||||
self.add_behaviour(commands_behaviour)
|
||||
self.add_behaviour(self.SendSpadeCommandsBehaviour())
|
||||
await self.add_background_task(self._zmq_command_loop())
|
||||
|
||||
self.logger.info("Finished setting up %s", self.jid)
|
||||
self.logger.info("Finished setting up %s", self.name)
|
||||
|
||||
async def stop(self):
|
||||
if self.subsocket:
|
||||
self.subsocket.close()
|
||||
if self.pubsocket:
|
||||
self.pubsocket.close()
|
||||
await super().stop()
|
||||
|
||||
async def handle_message(self, msg: InternalMessage):
|
||||
"""
|
||||
Handle commands received from other Python agents.
|
||||
"""
|
||||
try:
|
||||
speech_command = SpeechCommand.model_validate_json(msg.body)
|
||||
await self.pubsocket.send_json(speech_command.model_dump())
|
||||
except Exception:
|
||||
self.logger.exception("Error processing internal message.")
|
||||
|
||||
async def _zmq_command_loop(self):
|
||||
"""
|
||||
Handle commands from the UI.
|
||||
"""
|
||||
while self._running:
|
||||
try:
|
||||
_, body = await self.subsocket.recv_multipart()
|
||||
|
||||
body = json.loads(body)
|
||||
message = SpeechCommand.model_validate(body)
|
||||
|
||||
await self.pubsocket.send_json(message.model_dump())
|
||||
except Exception:
|
||||
self.logger.exception("Error processing ZMQ message.")
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import logging
|
||||
|
||||
from spade.agent import Agent
|
||||
from control_backend.core.agent_system import BaseAgent as CoreBaseAgent
|
||||
|
||||
|
||||
class BaseAgent(Agent):
|
||||
class BaseAgent(CoreBaseAgent):
|
||||
"""
|
||||
Base agent class for our agents to inherit from.
|
||||
This ensures that all agents have a logger.
|
||||
"""
|
||||
|
||||
logger: logging.Logger
|
||||
|
||||
@@ -1,67 +1,121 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
|
||||
import agentspeak
|
||||
from spade.behaviour import OneShotBehaviour
|
||||
from spade.message import Message
|
||||
from spade_bdi.bdi import BDIAgent
|
||||
import agentspeak.runtime
|
||||
import agentspeak.stdlib
|
||||
|
||||
from control_backend.agents.base import BaseAgent
|
||||
from control_backend.core.agent_system import InternalMessage
|
||||
from control_backend.core.config import settings
|
||||
|
||||
from .behaviours.belief_setter_behaviour import BeliefSetterBehaviour
|
||||
from .behaviours.receive_llm_resp_behaviour import ReceiveLLMResponseBehaviour
|
||||
from control_backend.schemas.ri_message import SpeechCommand
|
||||
|
||||
|
||||
class BDICoreAgent(BDIAgent):
|
||||
"""
|
||||
This is the Brain agent that does the belief inference with AgentSpeak.
|
||||
This is a continous process that happens automatically in the background.
|
||||
This class contains all the actions that can be called from AgentSpeak plans.
|
||||
It has the BeliefSetter behaviour and can aks and recieve requests from the LLM agent.
|
||||
"""
|
||||
|
||||
logger = logging.getLogger(__package__).getChild(__name__)
|
||||
class BDICoreAgent(BaseAgent):
|
||||
def __init__(self, name: str, asl: str):
|
||||
super().__init__(name)
|
||||
self.asl_file = asl
|
||||
self.env = agentspeak.runtime.Environment()
|
||||
self.bdi_agent = None
|
||||
self.actions = agentspeak.stdlib.actions
|
||||
|
||||
async def setup(self) -> None:
|
||||
"""
|
||||
Initializes belief behaviors and message routing.
|
||||
"""
|
||||
self.logger.info("BDICoreAgent setup started.")
|
||||
self.logger.debug("Setup started.")
|
||||
|
||||
self.add_behaviour(BeliefSetterBehaviour())
|
||||
self.add_behaviour(ReceiveLLMResponseBehaviour())
|
||||
self._add_custom_actions()
|
||||
|
||||
self.logger.info("BDICoreAgent setup complete.")
|
||||
await self._load_asl()
|
||||
|
||||
def add_custom_actions(self, actions) -> None:
|
||||
# Start the BDI cycle loop
|
||||
await self.add_background_task(self._bdi_loop())
|
||||
self.logger.debug("Setup complete.")
|
||||
|
||||
async def _load_asl(self):
|
||||
try:
|
||||
with open(self.asl_file) as source:
|
||||
self.bdi_agent = self.env.build_agent(source, self.actions)
|
||||
except FileNotFoundError:
|
||||
self.logger.warning(f"Could not find the specified ASL file at {self.asl_file}.")
|
||||
self.bdi_agent = agentspeak.runtime.Agent(self.env, self.name)
|
||||
|
||||
async def _bdi_loop(self):
|
||||
"""Runs the AgentSpeak BDI loop."""
|
||||
while self._running:
|
||||
assert self.bdi_agent is not None
|
||||
self.bdi_agent.step()
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def handle_message(self, msg: InternalMessage):
|
||||
"""
|
||||
Registers custom AgentSpeak actions callable from plans.
|
||||
Route incoming messages (Beliefs or LLM responses).
|
||||
"""
|
||||
sender = msg.sender
|
||||
|
||||
@actions.add(".reply", 1)
|
||||
def _reply(agent: "BDICoreAgent", term, intention):
|
||||
match sender:
|
||||
case settings.agent_settings.bdi_belief_collector_name:
|
||||
self.logger.debug("Processing message from belief collector.")
|
||||
try:
|
||||
if msg.thread == "beliefs":
|
||||
beliefs = json.loads(msg.body)
|
||||
self._add_beliefs(beliefs)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error processing belief: {e}")
|
||||
case settings.agent_settings.llm_name:
|
||||
content = msg.body
|
||||
self.logger.info("Received LLM response: %s", content)
|
||||
|
||||
# Forward to Robot Speech Agent
|
||||
cmd = SpeechCommand(data=content)
|
||||
out_msg = InternalMessage(
|
||||
to=settings.agent_settings.robot_speech_name,
|
||||
sender=self.name,
|
||||
body=cmd.model_dump_json(),
|
||||
)
|
||||
await self.send(out_msg)
|
||||
|
||||
# TODO: test way of adding beliefs
|
||||
def _add_beliefs(self, beliefs: dict[str, list[str]]):
|
||||
if not beliefs:
|
||||
return
|
||||
|
||||
for belief_name, args in beliefs.items():
|
||||
self._add_belief(belief_name, args)
|
||||
|
||||
if belief_name == "user_said":
|
||||
self._add_belief("user_said")
|
||||
|
||||
def _add_belief(self, belief_name: str, arguments: Iterable[str] = []):
|
||||
args = (agentspeak.Literal(arg) for arg in arguments)
|
||||
literal_belief = agentspeak.Literal(belief_name, args)
|
||||
|
||||
assert self.bdi_agent is not None
|
||||
|
||||
self.bdi_agent.call(
|
||||
agentspeak.Trigger.addition,
|
||||
agentspeak.GoalType.belief,
|
||||
literal_belief,
|
||||
agentspeak.runtime.Intention(),
|
||||
)
|
||||
self.logger.debug(f"Added belief {belief_name}({','.join(arguments)})")
|
||||
|
||||
def _add_custom_actions(self) -> None:
|
||||
"""Add any custom actions here."""
|
||||
|
||||
@self.actions.add(".reply", 1)
|
||||
def _reply(agent, term, intention):
|
||||
"""
|
||||
Sends text to the LLM (AgentSpeak action).
|
||||
Example: .reply("Hello LLM!")
|
||||
Sends text to the LLM.
|
||||
"""
|
||||
message_text = agentspeak.grounded(term.args[0], intention.scope)
|
||||
self.logger.debug("Reply action sending: %s", message_text)
|
||||
|
||||
self._send_to_llm(str(message_text))
|
||||
asyncio.create_task(self._send_to_llm(str(message_text)))
|
||||
yield
|
||||
|
||||
def _send_to_llm(self, text: str):
|
||||
async def _send_to_llm(self, text: str):
|
||||
"""
|
||||
Sends a text query to the LLM Agent asynchronously.
|
||||
Sends a text query to the LLM agent asynchronously.
|
||||
"""
|
||||
|
||||
class SendBehaviour(OneShotBehaviour):
|
||||
async def run(self) -> None:
|
||||
msg = Message(
|
||||
to=settings.agent_settings.llm_name + "@" + settings.agent_settings.host,
|
||||
body=text,
|
||||
)
|
||||
|
||||
await self.send(msg)
|
||||
self.agent.logger.info("Message sent to LLM agent: %s", text)
|
||||
|
||||
self.add_behaviour(SendBehaviour())
|
||||
msg = InternalMessage(to=settings.agent_settings.llm_name, sender=self.name, body=text)
|
||||
await self.send(msg)
|
||||
self.logger.info("Message sent to LLM agent: %s", text)
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
import json
|
||||
|
||||
from spade.agent import Message
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
from spade_bdi.bdi import BDIAgent
|
||||
|
||||
from control_backend.core.config import settings
|
||||
|
||||
|
||||
class BeliefSetterBehaviour(CyclicBehaviour):
|
||||
"""
|
||||
This is the behaviour that the BDI agent runs. This behaviour waits for incoming
|
||||
message and updates the agent's beliefs accordingly.
|
||||
"""
|
||||
|
||||
agent: BDIAgent
|
||||
|
||||
async def run(self):
|
||||
"""Polls for messages and processes them."""
|
||||
msg = await self.receive()
|
||||
self.agent.logger.debug(
|
||||
"Received message from %s with thread '%s' and body: %s",
|
||||
msg.sender,
|
||||
msg.thread,
|
||||
msg.body,
|
||||
)
|
||||
self._process_message(msg)
|
||||
|
||||
def _process_message(self, message: Message):
|
||||
"""Routes the message to the correct processing function based on the sender."""
|
||||
sender = message.sender.node # removes host from jid and converts to str
|
||||
self.agent.logger.debug("Processing message from sender: %s", sender)
|
||||
|
||||
match sender:
|
||||
case settings.agent_settings.bdi_belief_collector_name:
|
||||
self.agent.logger.debug(
|
||||
"Message is from the belief collector agent. Processing as belief message."
|
||||
)
|
||||
self._process_belief_message(message)
|
||||
case _:
|
||||
self.agent.logger.debug("Not the belief agent, discarding message")
|
||||
pass
|
||||
|
||||
def _process_belief_message(self, message: Message):
|
||||
if not message.body:
|
||||
self.agent.logger.debug("Ignoring message with empty body from %s", message.sender.node)
|
||||
return
|
||||
|
||||
match message.thread:
|
||||
case "beliefs":
|
||||
try:
|
||||
beliefs: dict[str, list[str]] = json.loads(message.body)
|
||||
self._set_beliefs(beliefs)
|
||||
except json.JSONDecodeError:
|
||||
self.agent.logger.error(
|
||||
"Could not decode beliefs from JSON. Message body: '%s'",
|
||||
message.body,
|
||||
exc_info=True,
|
||||
)
|
||||
case _:
|
||||
pass
|
||||
|
||||
def _set_beliefs(self, beliefs: dict[str, list[str]]):
|
||||
"""Removes previous values for beliefs and updates them with the provided values."""
|
||||
if self.agent.bdi is None:
|
||||
self.agent.logger.warning("Cannot set beliefs; agent's BDI is not yet initialized.")
|
||||
return
|
||||
|
||||
if not beliefs:
|
||||
self.agent.logger.debug("Received an empty set of beliefs. No beliefs were updated.")
|
||||
return
|
||||
|
||||
# Set new beliefs (outdated beliefs are automatically removed)
|
||||
for belief, arguments in beliefs.items():
|
||||
self.agent.logger.debug("Setting belief %s with arguments %s", belief, arguments)
|
||||
self.agent.bdi.set_belief(belief, *arguments)
|
||||
|
||||
# Special case: if there's a new user message, flag that we haven't responded yet
|
||||
if belief == "user_said":
|
||||
self.agent.bdi.set_belief("new_message")
|
||||
self.agent.logger.debug(
|
||||
"Detected 'user_said' belief, also setting 'new_message' belief."
|
||||
)
|
||||
|
||||
self.agent.logger.info("Successfully updated %d beliefs.", len(beliefs))
|
||||
@@ -1,37 +0,0 @@
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
from spade.message import Message
|
||||
|
||||
from control_backend.core.config import settings
|
||||
from control_backend.schemas.ri_message import SpeechCommand
|
||||
|
||||
|
||||
class ReceiveLLMResponseBehaviour(CyclicBehaviour):
|
||||
"""
|
||||
Adds behavior to receive responses from the LLM Agent.
|
||||
"""
|
||||
|
||||
async def run(self):
|
||||
msg = await self.receive()
|
||||
|
||||
sender = msg.sender.node
|
||||
match sender:
|
||||
case settings.agent_settings.llm_name:
|
||||
content = msg.body
|
||||
self.agent.logger.info("Received LLM response: %s", content)
|
||||
|
||||
speech_command = SpeechCommand(data=content)
|
||||
|
||||
message = Message(
|
||||
to=settings.agent_settings.robot_speech_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host,
|
||||
sender=self.agent.jid,
|
||||
body=speech_command.model_dump_json(),
|
||||
)
|
||||
|
||||
self.agent.logger.debug("Sending message: %s", message)
|
||||
|
||||
await self.send(message)
|
||||
case _:
|
||||
self.agent.logger.debug("Discarding message from %s", sender)
|
||||
pass
|
||||
@@ -1,92 +0,0 @@
|
||||
import json
|
||||
from json import JSONDecodeError
|
||||
|
||||
from spade.agent import Message
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
|
||||
from control_backend.core.config import settings
|
||||
|
||||
|
||||
class BeliefCollectorBehaviour(CyclicBehaviour):
|
||||
"""
|
||||
Continuously collects beliefs/emotions from extractor agents:
|
||||
Then we send a unified belief packet to the BDI agent.
|
||||
"""
|
||||
|
||||
async def run(self):
|
||||
msg = await self.receive()
|
||||
await self._process_message(msg)
|
||||
|
||||
async def _process_message(self, msg: Message):
|
||||
sender_node = msg.sender.node
|
||||
|
||||
# Parse JSON payload
|
||||
try:
|
||||
payload = json.loads(msg.body)
|
||||
except JSONDecodeError as e:
|
||||
self.agent.logger.warning(
|
||||
"BeliefCollector: failed to parse JSON from %s. Body=%r Error=%s",
|
||||
sender_node,
|
||||
msg.body,
|
||||
e,
|
||||
)
|
||||
return
|
||||
|
||||
msg_type = payload.get("type")
|
||||
|
||||
# Prefer explicit 'type' field
|
||||
if msg_type == "belief_extraction_text" or sender_node == "bel_text_agent_mock":
|
||||
self.agent.logger.debug(
|
||||
"Message routed to _handle_belief_text (sender=%s)", sender_node
|
||||
)
|
||||
await self._handle_belief_text(payload, sender_node)
|
||||
# This is not implemented yet, but we keep the structure for future use
|
||||
elif msg_type == "emotion_extraction_text" or sender_node == "emo_text_agent_mock":
|
||||
self.agent.logger.debug("Message routed to _handle_emo_text (sender=%s)", sender_node)
|
||||
await self._handle_emo_text(payload, sender_node)
|
||||
else:
|
||||
self.agent.logger.warning(
|
||||
"Unrecognized message (sender=%s, type=%r). Ignoring.", sender_node, msg_type
|
||||
)
|
||||
|
||||
async def _handle_belief_text(self, payload: dict, origin: str):
|
||||
"""
|
||||
Expected payload:
|
||||
{
|
||||
"type": "belief_extraction_text",
|
||||
"beliefs": {"user_said": ["Can you help me?"]}
|
||||
|
||||
}
|
||||
|
||||
"""
|
||||
beliefs = payload.get("beliefs", {})
|
||||
|
||||
if not beliefs:
|
||||
self.agent.logger.debug("Received empty beliefs set.")
|
||||
return
|
||||
|
||||
self.agent.logger.debug("Forwarding %d beliefs.", len(beliefs))
|
||||
for belief_name, belief_list in beliefs.items():
|
||||
for belief in belief_list:
|
||||
self.agent.logger.debug(" - %s %s", belief_name, str(belief))
|
||||
|
||||
await self._send_beliefs_to_bdi(beliefs, origin=origin)
|
||||
|
||||
async def _handle_emo_text(self, payload: dict, origin: str):
|
||||
"""TODO: implement (after we have emotional recogntion)"""
|
||||
pass
|
||||
|
||||
async def _send_beliefs_to_bdi(self, beliefs: list[str], origin: str | None = None):
|
||||
"""
|
||||
Sends a unified belief packet to the BDI agent.
|
||||
"""
|
||||
if not beliefs:
|
||||
return
|
||||
|
||||
to_jid = f"{settings.agent_settings.bdi_core_name}@{settings.agent_settings.host}"
|
||||
|
||||
msg = Message(to=to_jid, sender=self.agent.jid, thread="beliefs")
|
||||
msg.body = json.dumps(beliefs)
|
||||
|
||||
await self.send(msg)
|
||||
self.agent.logger.info("Sent %d belief(s) to BDI core.", len(beliefs))
|
||||
@@ -1,11 +1,88 @@
|
||||
from control_backend.agents.base import BaseAgent
|
||||
import json
|
||||
|
||||
from .behaviours.belief_collector_behaviour import BeliefCollectorBehaviour
|
||||
from control_backend.agents.base import BaseAgent
|
||||
from control_backend.core.agent_system import InternalMessage
|
||||
from control_backend.core.config import settings
|
||||
|
||||
|
||||
class BDIBeliefCollectorAgent(BaseAgent):
|
||||
"""
|
||||
Continuously collects beliefs/emotions from extractor agents and forwards a
|
||||
unified belief packet to the BDI agent.
|
||||
"""
|
||||
|
||||
async def setup(self):
|
||||
self.logger.info("BDIBeliefCollectorAgent starting (%s)", self.jid)
|
||||
# Attach the continuous collector behaviour (listens and forwards to BDI)
|
||||
self.add_behaviour(BeliefCollectorBehaviour())
|
||||
self.logger.info("BDIBeliefCollectorAgent ready.")
|
||||
self.logger.info("Setting up %s", self.name)
|
||||
|
||||
async def handle_message(self, msg: InternalMessage):
|
||||
sender_node = msg.sender
|
||||
|
||||
# Parse JSON payload
|
||||
try:
|
||||
payload = json.loads(msg.body)
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
"BeliefCollector: failed to parse JSON from %s. Body=%r Error=%s",
|
||||
sender_node,
|
||||
msg.body,
|
||||
e,
|
||||
)
|
||||
return
|
||||
|
||||
msg_type = payload.get("type")
|
||||
|
||||
# Prefer explicit 'type' field
|
||||
if msg_type == "belief_extraction_text":
|
||||
self.logger.debug("Message routed to _handle_belief_text (sender=%s)", sender_node)
|
||||
await self._handle_belief_text(payload, sender_node)
|
||||
# This is not implemented yet, but we keep the structure for future use
|
||||
elif msg_type == "emotion_extraction_text":
|
||||
self.logger.debug("Message routed to _handle_emo_text (sender=%s)", sender_node)
|
||||
await self._handle_emo_text(payload, sender_node)
|
||||
else:
|
||||
self.logger.warning(
|
||||
"Unrecognized message (sender=%s, type=%r). Ignoring.", sender_node, msg_type
|
||||
)
|
||||
|
||||
async def _handle_belief_text(self, payload: dict, origin: str):
|
||||
"""
|
||||
Expected payload:
|
||||
{
|
||||
"type": "belief_extraction_text",
|
||||
"beliefs": {"user_said": ["Can you help me?"]}
|
||||
|
||||
}
|
||||
"""
|
||||
beliefs = payload.get("beliefs", {})
|
||||
|
||||
if not beliefs:
|
||||
self.logger.debug("Received empty beliefs set.")
|
||||
return
|
||||
|
||||
self.logger.debug("Forwarding %d beliefs.", len(beliefs))
|
||||
for belief_name, belief_list in beliefs.items():
|
||||
for belief in belief_list:
|
||||
self.logger.debug(" - %s %s", belief_name, str(belief))
|
||||
|
||||
await self._send_beliefs_to_bdi(beliefs, origin=origin)
|
||||
|
||||
async def _handle_emo_text(self, payload: dict, origin: str):
|
||||
"""TODO: implement (after we have emotional recognition)"""
|
||||
pass
|
||||
|
||||
async def _send_beliefs_to_bdi(self, beliefs: dict, origin: str | None = None):
|
||||
"""
|
||||
Sends a unified belief packet to the BDI agent.
|
||||
"""
|
||||
if not beliefs:
|
||||
return
|
||||
|
||||
msg = InternalMessage(
|
||||
to=settings.agent_settings.bdi_core_name,
|
||||
sender=self.name,
|
||||
body=json.dumps(beliefs),
|
||||
thread="beliefs",
|
||||
)
|
||||
|
||||
await self.send(msg)
|
||||
self.logger.info("Sent %d belief(s) to BDI core.", len(beliefs))
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
from spade.message import Message
|
||||
|
||||
from control_backend.core.config import settings
|
||||
|
||||
|
||||
class TextBeliefExtractorBehaviour(CyclicBehaviour):
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# TODO: LLM prompt nog hardcoded
|
||||
llm_instruction_prompt = """
|
||||
You are an information extraction assistent for a BDI agent. Your task is to extract values \
|
||||
from a user's text to bind a list of ungrounded beliefs. Rules:
|
||||
You will receive a JSON object with "beliefs" (a list of ungrounded AgentSpeak beliefs) \
|
||||
and "text" (user's transcript).
|
||||
Analyze the text to find values that sematically match the variables (X,Y,Z) in the beliefs.
|
||||
A single piece of text might contain multiple instances that match a belief.
|
||||
Respond ONLY with a single JSON object.
|
||||
The JSON object's keys should be the belief functors (e.g., "weather").
|
||||
The value for each key must be a list of lists.
|
||||
Each inner list must contain the extracted arguments (as strings) for one instance \
|
||||
of that belief.
|
||||
CRITICAL: If no information in the text matches a belief, DO NOT include that key \
|
||||
in your response.
|
||||
"""
|
||||
|
||||
# on_start agent receives message containing the beliefs to look out for and
|
||||
# sets up the LLM with instruction prompt
|
||||
# async def on_start(self):
|
||||
# msg = await self.receive(timeout=0.1)
|
||||
# self.beliefs = dict uit message
|
||||
# send instruction prompt to LLM
|
||||
|
||||
beliefs: dict[str, list[str]]
|
||||
beliefs = {"mood": ["X"], "car": ["Y"]}
|
||||
|
||||
async def run(self):
|
||||
msg = await self.receive()
|
||||
if msg is None:
|
||||
return
|
||||
|
||||
sender = msg.sender.node
|
||||
match sender:
|
||||
case settings.agent_settings.transcription_name:
|
||||
self.logger.debug("Received text from transcriber: %s", msg.body)
|
||||
await self._process_transcription_demo(msg.body)
|
||||
case _:
|
||||
self.logger.info("Discarding message from %s", sender)
|
||||
pass
|
||||
|
||||
async def _process_transcription(self, text: str):
|
||||
text_prompt = f"Text: {text}"
|
||||
|
||||
beliefs_prompt = "These are the beliefs to be bound:\n"
|
||||
for belief, values in self.beliefs.items():
|
||||
beliefs_prompt += f"{belief}({', '.join(values)})\n"
|
||||
|
||||
prompt = text_prompt + beliefs_prompt
|
||||
self.logger.info(prompt)
|
||||
# prompt_msg = Message(to="LLMAgent@whatever")
|
||||
# response = self.send(prompt_msg)
|
||||
|
||||
# Mock response; response is beliefs in JSON format, it parses do dict[str,list[list[str]]]
|
||||
response = '{"mood": [["happy"]]}'
|
||||
# Verify by trying to parse
|
||||
try:
|
||||
json.loads(response)
|
||||
belief_message = Message()
|
||||
|
||||
belief_message.to = (
|
||||
settings.agent_settings.bdi_belief_collector_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host
|
||||
)
|
||||
belief_message.body = response
|
||||
belief_message.thread = "beliefs"
|
||||
|
||||
await self.send(belief_message)
|
||||
self.agent.logger.info("Sent beliefs to BDI.")
|
||||
except json.JSONDecodeError:
|
||||
# Parsing failed, so the response is in the wrong format, log warning
|
||||
self.agent.logger.warning("Received LLM response in incorrect format.")
|
||||
|
||||
async def _process_transcription_demo(self, txt: str):
|
||||
"""
|
||||
Demo version to process the transcription input to beliefs. For the demo only the belief
|
||||
'user_said' is relevant, so this function simply makes a dict with key: "user_said",
|
||||
value: txt and passes this to the Belief Collector agent.
|
||||
"""
|
||||
belief = {"beliefs": {"user_said": [txt]}, "type": "belief_extraction_text"}
|
||||
payload = json.dumps(belief)
|
||||
belief_msg = Message()
|
||||
|
||||
belief_msg.to = (
|
||||
settings.agent_settings.bdi_belief_collector_name + "@" + settings.agent_settings.host
|
||||
)
|
||||
belief_msg.body = payload
|
||||
belief_msg.thread = "beliefs"
|
||||
|
||||
await self.send(belief_msg)
|
||||
self.logger.info("Sent %d beliefs to the belief collector.", len(belief["beliefs"]))
|
||||
@@ -1,8 +1,38 @@
|
||||
from control_backend.agents.base import BaseAgent
|
||||
import json
|
||||
|
||||
from .behaviours.text_belief_extractor_behaviour import TextBeliefExtractorBehaviour
|
||||
from control_backend.agents.base import BaseAgent
|
||||
from control_backend.core.agent_system import InternalMessage
|
||||
from control_backend.core.config import settings
|
||||
|
||||
|
||||
class TextBeliefExtractorAgent(BaseAgent):
|
||||
async def setup(self):
|
||||
self.add_behaviour(TextBeliefExtractorBehaviour())
|
||||
self.logger.info("Settting up %s.", self.name)
|
||||
# Setup LLM belief context if needed (currently demo is just passthrough)
|
||||
self.beliefs = {"mood": ["X"], "car": ["Y"]}
|
||||
|
||||
async def handle_message(self, msg: InternalMessage):
|
||||
sender = msg.sender
|
||||
if sender == settings.agent_settings.transcription_name:
|
||||
self.logger.debug("Received text from transcriber: %s", msg.body)
|
||||
await self._process_transcription_demo(msg.body)
|
||||
else:
|
||||
self.logger.info("Discarding message from %s", sender)
|
||||
|
||||
async def _process_transcription_demo(self, txt: str):
|
||||
"""
|
||||
Demo version to process the transcription input to beliefs.
|
||||
"""
|
||||
# For demo, just wrapping user text as user_said belief
|
||||
belief = {"beliefs": {"user_said": [txt]}, "type": "belief_extraction_text"}
|
||||
payload = json.dumps(belief)
|
||||
|
||||
belief_msg = InternalMessage(
|
||||
to=settings.agent_settings.bdi_belief_collector_name,
|
||||
sender=self.name,
|
||||
body=payload,
|
||||
thread="beliefs",
|
||||
)
|
||||
|
||||
await self.send(belief_msg)
|
||||
self.logger.info("Sent %d beliefs to the belief collector.", len(belief["beliefs"]))
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import zmq.asyncio
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
import zmq
|
||||
import zmq.asyncio as azmq
|
||||
from zmq.asyncio import Context
|
||||
|
||||
from control_backend.agents import BaseAgent
|
||||
@@ -12,109 +12,38 @@ from ..actuation.robot_speech_agent import RobotSpeechAgent
|
||||
|
||||
|
||||
class RICommunicationAgent(BaseAgent):
|
||||
req_socket: zmq.Socket
|
||||
_address = ""
|
||||
_bind = True
|
||||
connected = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
jid: str,
|
||||
password: str,
|
||||
port: int = settings.agent_settings.default_spade_port,
|
||||
verify_security: bool = False,
|
||||
name: str,
|
||||
address=settings.zmq_settings.ri_command_address,
|
||||
bind=False,
|
||||
):
|
||||
super().__init__(jid, password, port, verify_security)
|
||||
super().__init__(name)
|
||||
self._address = address
|
||||
self._bind = bind
|
||||
self._req_socket: zmq.asyncio.Socket | None = None
|
||||
self.pub_socket: zmq.asyncio.Socket | None = None
|
||||
self._req_socket: azmq.Socket | None = None
|
||||
self.pub_socket: azmq.Socket | None = None
|
||||
self.connected = False
|
||||
|
||||
class ListenBehaviour(CyclicBehaviour):
|
||||
async def run(self):
|
||||
"""
|
||||
Run the listening (ping) loop indefinetely.
|
||||
"""
|
||||
assert self.agent is not None
|
||||
async def setup(self):
|
||||
"""
|
||||
Try to set up the communication agent, we have `behaviour_settings.comm_setup_max_retries`
|
||||
retries in case we don't have a response yet.
|
||||
"""
|
||||
self.logger.info("Setting up %s", self.name)
|
||||
|
||||
if not self.agent.connected:
|
||||
await asyncio.sleep(settings.behaviour_settings.sleep_s)
|
||||
return
|
||||
# Bind request socket
|
||||
await self._setup_sockets()
|
||||
|
||||
# We need to listen and sent pings.
|
||||
message = {"endpoint": "ping", "data": {"id": "e.g. some reference id"}}
|
||||
seconds_to_wait_total = settings.behaviour_settings.sleep_s
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self.agent._req_socket.send_json(message), timeout=seconds_to_wait_total / 2
|
||||
)
|
||||
except TimeoutError:
|
||||
self.agent.logger.debug(
|
||||
"Waited too long to send message - "
|
||||
"we probably dont have any receivers... but let's check!"
|
||||
)
|
||||
if await self._negotiate_connection():
|
||||
self.connected = True
|
||||
await self.add_background_task(self._listen_loop())
|
||||
else:
|
||||
self.logger.warning("Failed to negotiate connection during setup.")
|
||||
|
||||
# Wait up to {seconds_to_wait_total/2} seconds for a reply
|
||||
try:
|
||||
message = await asyncio.wait_for(
|
||||
self.agent._req_socket.recv_json(), timeout=seconds_to_wait_total / 2
|
||||
)
|
||||
self.logger.info("Finished setting up %s", self.name)
|
||||
|
||||
# We didnt get a reply
|
||||
except TimeoutError:
|
||||
self.agent.logger.info(
|
||||
f"No ping retrieved in {seconds_to_wait_total} seconds, "
|
||||
"sending UI disconnection event and attempting to restart."
|
||||
)
|
||||
|
||||
# Make sure we dont retry receiving messages untill we're setup.
|
||||
self.agent.connected = False
|
||||
self.agent.remove_behaviour(self)
|
||||
|
||||
# Tell UI we're disconnected.
|
||||
topic = b"ping"
|
||||
data = json.dumps(False).encode()
|
||||
if self.agent.pub_socket is None:
|
||||
self.agent.logger.warning(
|
||||
"Communication agent pub socket not correctly initialized."
|
||||
)
|
||||
else:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self.agent.pub_socket.send_multipart([topic, data]), 5
|
||||
)
|
||||
except TimeoutError:
|
||||
self.agent.logger.warning(
|
||||
f"Initial connection ping for router timed out in {self.agent.name}."
|
||||
)
|
||||
|
||||
# Try to reboot.
|
||||
self.agent.logger.debug("Restarting communication agent.")
|
||||
await self.agent.setup()
|
||||
|
||||
self.agent.logger.debug(f'Received message "{message}" from RI.')
|
||||
if "endpoint" not in message:
|
||||
self.agent.logger.warning(
|
||||
"No received endpoint in message, expected ping endpoint."
|
||||
)
|
||||
return
|
||||
|
||||
# See what endpoint we received
|
||||
match message["endpoint"]:
|
||||
case "ping":
|
||||
topic = b"ping"
|
||||
data = json.dumps(True).encode()
|
||||
if self.agent.pub_socket is not None:
|
||||
await self.agent.pub_socket.send_multipart([topic, data])
|
||||
await asyncio.sleep(settings.behaviour_settings.sleep_s)
|
||||
case _:
|
||||
self.agent.logger.debug(
|
||||
"Received message with topic different than ping, while ping expected."
|
||||
)
|
||||
|
||||
async def setup_sockets(self, force=False):
|
||||
async def _setup_sockets(self, force=False):
|
||||
"""
|
||||
Sets up request socket for communication agent.
|
||||
"""
|
||||
@@ -130,21 +59,13 @@ class RICommunicationAgent(BaseAgent):
|
||||
self.pub_socket = Context.instance().socket(zmq.PUB)
|
||||
self.pub_socket.connect(settings.zmq_settings.internal_pub_address)
|
||||
|
||||
async def setup(self, max_retries: int = settings.behaviour_settings.comm_setup_max_retries):
|
||||
"""
|
||||
Try to set up the communication agent, we have `behaviour_settings.comm_setup_max_retries`
|
||||
retries in case we don't have a response yet.
|
||||
"""
|
||||
self.logger.info("Setting up %s", self.jid)
|
||||
|
||||
# Bind request socket
|
||||
await self.setup_sockets()
|
||||
|
||||
async def _negotiate_connection(
|
||||
self, max_retries: int = settings.behaviour_settings.comm_setup_max_retries
|
||||
):
|
||||
retries = 0
|
||||
# Let's try a certain amount of times before failing connection
|
||||
while retries < max_retries:
|
||||
# Make sure the socket is properly setup.
|
||||
if self._req_socket is None:
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
# Send our message and receive one back
|
||||
@@ -156,7 +77,6 @@ class RICommunicationAgent(BaseAgent):
|
||||
received_message = await asyncio.wait_for(
|
||||
self._req_socket.recv_json(), timeout=retry_frequency
|
||||
)
|
||||
|
||||
except TimeoutError:
|
||||
self.logger.warning(
|
||||
"No connection established in %d seconds (attempt %d/%d)",
|
||||
@@ -166,7 +86,6 @@ class RICommunicationAgent(BaseAgent):
|
||||
)
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning("Unexpected error during negotiation: %s", e)
|
||||
retries += 1
|
||||
@@ -187,64 +106,129 @@ class RICommunicationAgent(BaseAgent):
|
||||
|
||||
# At this point, we have a valid response
|
||||
try:
|
||||
for port_data in received_message["data"]:
|
||||
id = port_data["id"]
|
||||
port = port_data["port"]
|
||||
bind = port_data["bind"]
|
||||
|
||||
if not bind:
|
||||
addr = f"tcp://localhost:{port}"
|
||||
else:
|
||||
addr = f"tcp://*:{port}"
|
||||
|
||||
match id:
|
||||
case "main":
|
||||
if addr != self._address:
|
||||
if not bind:
|
||||
self._req_socket.connect(addr)
|
||||
else:
|
||||
self._req_socket.bind(addr)
|
||||
case "actuation":
|
||||
ri_commands_agent = RobotSpeechAgent(
|
||||
settings.agent_settings.robot_speech_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host,
|
||||
settings.agent_settings.robot_speech_name,
|
||||
address=addr,
|
||||
bind=bind,
|
||||
)
|
||||
await ri_commands_agent.start()
|
||||
case _:
|
||||
self.logger.warning("Unhandled negotiation id: %s", id)
|
||||
|
||||
await self._handle_negotiation_response(received_message)
|
||||
# Let UI know that we're connected
|
||||
topic = b"ping"
|
||||
data = json.dumps(True).encode()
|
||||
if self.pub_socket:
|
||||
await self.pub_socket.send_multipart([topic, data])
|
||||
return True
|
||||
except Exception as e:
|
||||
self.logger.warning("Error unpacking negotiation data: %s", e)
|
||||
retries += 1
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
|
||||
# setup succeeded
|
||||
break
|
||||
return False
|
||||
|
||||
else:
|
||||
self.logger.warning("Failed to set up %s after %d retries", self.name, max_retries)
|
||||
return
|
||||
async def _handle_negotiation_response(self, received_message):
|
||||
for port_data in received_message["data"]:
|
||||
id = port_data["id"]
|
||||
port = port_data["port"]
|
||||
bind = port_data["bind"]
|
||||
|
||||
# Set up ping behaviour
|
||||
listen_behaviour = self.ListenBehaviour()
|
||||
self.add_behaviour(listen_behaviour)
|
||||
if not bind:
|
||||
addr = f"tcp://localhost:{port}"
|
||||
else:
|
||||
addr = f"tcp://*:{port}"
|
||||
|
||||
# Let UI know that we're connected
|
||||
match id:
|
||||
case "main":
|
||||
if addr != self._address:
|
||||
assert self._req_socket is not None
|
||||
if not bind:
|
||||
self._req_socket.connect(addr)
|
||||
else:
|
||||
self._req_socket.bind(addr)
|
||||
case "actuation":
|
||||
ri_commands_agent = RobotSpeechAgent(
|
||||
settings.agent_settings.robot_speech_name,
|
||||
address=addr,
|
||||
bind=bind,
|
||||
)
|
||||
await ri_commands_agent.start()
|
||||
case _:
|
||||
self.logger.warning("Unhandled negotiation id: %s", id)
|
||||
|
||||
async def stop(self):
|
||||
if self._req_socket:
|
||||
self._req_socket.close()
|
||||
if self.pub_socket:
|
||||
self.pub_socket.close()
|
||||
await super().stop()
|
||||
|
||||
async def _listen_loop(self):
|
||||
"""
|
||||
Run the listening (ping) loop indefinitely.
|
||||
"""
|
||||
while self._running:
|
||||
if not self.connected:
|
||||
await asyncio.sleep(settings.behaviour_settings.sleep_s)
|
||||
continue
|
||||
|
||||
# We need to listen and send pings.
|
||||
message = {"endpoint": "ping", "data": {"id": "e.g. some reference id"}}
|
||||
seconds_to_wait_total = settings.behaviour_settings.sleep_s
|
||||
try:
|
||||
assert self._req_socket is not None
|
||||
await asyncio.wait_for(
|
||||
self._req_socket.send_json(message), timeout=seconds_to_wait_total / 2
|
||||
)
|
||||
except TimeoutError:
|
||||
self.logger.debug(
|
||||
"Waited too long to send message - "
|
||||
"we probably dont have any receivers... but let's check!"
|
||||
)
|
||||
|
||||
# Wait up to {seconds_to_wait_total/2} seconds for a reply
|
||||
try:
|
||||
assert self._req_socket is not None
|
||||
message = await asyncio.wait_for(
|
||||
self._req_socket.recv_json(), timeout=seconds_to_wait_total / 2
|
||||
)
|
||||
|
||||
self.logger.debug(f'Received message "{message}" from RI.')
|
||||
if "endpoint" not in message:
|
||||
self.logger.warning("No received endpoint in message, expected ping endpoint.")
|
||||
continue
|
||||
|
||||
# See what endpoint we received
|
||||
match message["endpoint"]:
|
||||
case "ping":
|
||||
topic = b"ping"
|
||||
data = json.dumps(True).encode()
|
||||
if self.pub_socket is not None:
|
||||
await self.pub_socket.send_multipart([topic, data])
|
||||
await asyncio.sleep(settings.behaviour_settings.sleep_s)
|
||||
case _:
|
||||
self.logger.debug(
|
||||
"Received message with topic different than ping, while ping expected."
|
||||
)
|
||||
# We didnt get a reply
|
||||
except TimeoutError:
|
||||
self.logger.info(
|
||||
f"No ping retrieved in {seconds_to_wait_total} seconds, "
|
||||
"sending UI disconnection event and attempting to restart."
|
||||
)
|
||||
await self._handle_disconnection()
|
||||
continue
|
||||
except Exception:
|
||||
self.logger.error("Error while waiting for ping message.", exc_info=True)
|
||||
raise
|
||||
|
||||
async def _handle_disconnection(self):
|
||||
self.connected = False
|
||||
|
||||
# Tell UI we're disconnected.
|
||||
topic = b"ping"
|
||||
data = json.dumps(True).encode()
|
||||
if self.pub_socket is None:
|
||||
self.logger.warning("Communication agent pub socket not correctly initialized.")
|
||||
else:
|
||||
data = json.dumps(False).encode()
|
||||
if self.pub_socket:
|
||||
try:
|
||||
await asyncio.wait_for(self.pub_socket.send_multipart([topic, data]), 5)
|
||||
except TimeoutError:
|
||||
self.logger.warning("Initial connection ping for router timed out in com_ri_agent.")
|
||||
self.logger.warning("Connection ping for router timed out.")
|
||||
|
||||
# Make sure to start listening now that we're connected.
|
||||
self.connected = True
|
||||
self.logger.info("Finished setting up %s", self.jid)
|
||||
# Try to reboot/renegotiate
|
||||
self.logger.debug("Restarting communication negotiation.")
|
||||
if await self._negotiate_connection(max_retries=1):
|
||||
self.connected = True
|
||||
|
||||
@@ -3,10 +3,9 @@ import re
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
import httpx
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
from spade.message import Message
|
||||
|
||||
from control_backend.agents import BaseAgent
|
||||
from control_backend.core.agent_system import InternalMessage
|
||||
from control_backend.core.config import settings
|
||||
|
||||
from .llm_instructions import LLMInstructions
|
||||
@@ -19,143 +18,114 @@ class LLMAgent(BaseAgent):
|
||||
and responds with processed LLM output.
|
||||
"""
|
||||
|
||||
class ReceiveMessageBehaviour(CyclicBehaviour):
|
||||
"""
|
||||
Cyclic behaviour to continuously listen for incoming messages from
|
||||
the BDI Core Agent and handle them.
|
||||
"""
|
||||
|
||||
async def run(self):
|
||||
"""
|
||||
Receives SPADE messages and processes only those originating from the
|
||||
configured BDI agent.
|
||||
"""
|
||||
msg = await self.receive()
|
||||
|
||||
sender = msg.sender.node
|
||||
self.agent.logger.debug(
|
||||
"Received message: %s from %s",
|
||||
msg.body,
|
||||
sender,
|
||||
)
|
||||
|
||||
if sender == settings.agent_settings.bdi_core_name:
|
||||
self.agent.logger.debug("Processing message from BDI Core Agent")
|
||||
await self._process_bdi_message(msg)
|
||||
else:
|
||||
self.agent.logger.debug("Message ignored (not from BDI Core Agent)")
|
||||
|
||||
async def _process_bdi_message(self, message: Message):
|
||||
"""
|
||||
Forwards user text from the BDI to the LLM and replies with the generated text in chunks
|
||||
separated by punctuation.
|
||||
"""
|
||||
user_text = message.body
|
||||
# Consume the streaming generator and send a reply for every chunk
|
||||
async for chunk in self._query_llm(user_text):
|
||||
await self._reply(chunk)
|
||||
self.agent.logger.debug(
|
||||
"Finished processing BDI message. Response sent in chunks to BDI Core Agent."
|
||||
)
|
||||
|
||||
async def _reply(self, msg: str):
|
||||
"""
|
||||
Sends a response message back to the BDI Core Agent.
|
||||
"""
|
||||
reply = Message(
|
||||
to=settings.agent_settings.bdi_core_name + "@" + settings.agent_settings.host,
|
||||
body=msg,
|
||||
)
|
||||
await self.send(reply)
|
||||
|
||||
async def _query_llm(self, prompt: str) -> AsyncGenerator[str]:
|
||||
"""
|
||||
Sends a chat completion request to the local LLM service and streams the response by
|
||||
yielding fragments separated by punctuation like.
|
||||
|
||||
:param prompt: Input text prompt to pass to the LLM.
|
||||
:yield: Fragments of the LLM-generated content.
|
||||
"""
|
||||
instructions = LLMInstructions(
|
||||
"- Be friendly and respectful.\n"
|
||||
"- Make the conversation feel natural and engaging.\n"
|
||||
"- Speak like a pirate.\n"
|
||||
"- When the user asks what you can do, tell them.",
|
||||
"- Try to learn the user's name during conversation.\n"
|
||||
"- Suggest playing a game of asking yes or no questions where you think of a word "
|
||||
"and the user must guess it.",
|
||||
)
|
||||
messages = [
|
||||
{
|
||||
"role": "developer",
|
||||
"content": instructions.build_developer_instruction(),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
current_chunk = ""
|
||||
async for token in self._stream_query_llm(messages):
|
||||
current_chunk += token
|
||||
|
||||
# Stream the message in chunks separated by punctuation.
|
||||
# We include the delimiter in the emitted chunk for natural flow.
|
||||
pattern = re.compile(r".*?(?:,|;|:|—|–|\.{3}|…|\.|\?|!)\s*", re.DOTALL)
|
||||
for m in pattern.finditer(current_chunk):
|
||||
chunk = m.group(0)
|
||||
if chunk:
|
||||
yield current_chunk
|
||||
current_chunk = ""
|
||||
|
||||
# Yield any remaining tail
|
||||
if current_chunk:
|
||||
yield current_chunk
|
||||
except httpx.HTTPError as err:
|
||||
self.agent.logger.error("HTTP error.", exc_info=err)
|
||||
yield "LLM service unavailable."
|
||||
except Exception as err:
|
||||
self.agent.logger.error("Unexpected error.", exc_info=err)
|
||||
yield "Error processing the request."
|
||||
|
||||
async def _stream_query_llm(self, messages) -> AsyncGenerator[str]:
|
||||
"""Raises httpx.HTTPError when the API gives an error."""
|
||||
async with httpx.AsyncClient(timeout=None) as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
settings.llm_settings.local_llm_url,
|
||||
json={
|
||||
"model": settings.llm_settings.local_llm_model,
|
||||
"messages": messages,
|
||||
"temperature": 0.3,
|
||||
"stream": True,
|
||||
},
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
async for line in response.aiter_lines():
|
||||
if not line or not line.startswith("data: "):
|
||||
continue
|
||||
|
||||
data = line[len("data: ") :]
|
||||
if data.strip() == "[DONE]":
|
||||
break
|
||||
|
||||
try:
|
||||
event = json.loads(data)
|
||||
delta = event.get("choices", [{}])[0].get("delta", {}).get("content")
|
||||
if delta:
|
||||
yield delta
|
||||
except json.JSONDecodeError:
|
||||
self.agent.logger.error("Failed to parse LLM response: %s", data)
|
||||
|
||||
async def setup(self):
|
||||
self.logger.info("Setting up %s.", self.name)
|
||||
|
||||
async def handle_message(self, msg: InternalMessage):
|
||||
if msg.sender == settings.agent_settings.bdi_core_name:
|
||||
self.logger.debug("Processing message from BDI core.")
|
||||
await self._process_bdi_message(msg)
|
||||
else:
|
||||
self.logger.debug("Message ignored (not from BDI core.")
|
||||
|
||||
async def _process_bdi_message(self, message: InternalMessage):
|
||||
user_text = message.body
|
||||
async for chunk in self._query_llm(user_text):
|
||||
await self._send_reply(chunk)
|
||||
self.logger.debug(
|
||||
"Finished processing BDI message. Response sent in chunks to BDI core."
|
||||
)
|
||||
|
||||
async def _send_reply(self, msg: str):
|
||||
"""
|
||||
Sets up the SPADE behaviour to filter and process messages from the
|
||||
BDI Core Agent.
|
||||
Sends a response message back to the BDI Core Agent.
|
||||
"""
|
||||
behaviour = self.ReceiveMessageBehaviour()
|
||||
self.add_behaviour(behaviour)
|
||||
self.logger.info("LLMAgent setup complete")
|
||||
reply = InternalMessage(
|
||||
to=settings.agent_settings.bdi_core_name + "@" + settings.agent_settings.host,
|
||||
sender=self.name,
|
||||
body=msg,
|
||||
)
|
||||
await self.send(reply)
|
||||
|
||||
async def _query_llm(self, prompt: str) -> AsyncGenerator[str]:
|
||||
"""
|
||||
Sends a chat completion request to the local LLM service and streams the response by
|
||||
yielding fragments separated by punctuation like.
|
||||
|
||||
:param prompt: Input text prompt to pass to the LLM.
|
||||
:yield: Fragments of the LLM-generated content.
|
||||
"""
|
||||
instructions = LLMInstructions(
|
||||
"- Be friendly and respectful.\n"
|
||||
"- Make the conversation feel natural and engaging.\n"
|
||||
"- Speak like a pirate.\n"
|
||||
"- When the user asks what you can do, tell them.",
|
||||
"- Try to learn the user's name during conversation.\n"
|
||||
"- Suggest playing a game of asking yes or no questions where you think of a word "
|
||||
"and the user must guess it.",
|
||||
)
|
||||
messages = [
|
||||
{
|
||||
"role": "developer",
|
||||
"content": instructions.build_developer_instruction(),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
current_chunk = ""
|
||||
async for token in self._stream_query_llm(messages):
|
||||
current_chunk += token
|
||||
|
||||
# Stream the message in chunks separated by punctuation.
|
||||
# We include the delimiter in the emitted chunk for natural flow.
|
||||
pattern = re.compile(r".*?(?:,|;|:|—|–|\.{3}|…|\.|\?|!)\s*", re.DOTALL)
|
||||
for m in pattern.finditer(current_chunk):
|
||||
chunk = m.group(0)
|
||||
if chunk:
|
||||
yield current_chunk
|
||||
current_chunk = ""
|
||||
|
||||
# Yield any remaining tail
|
||||
if current_chunk:
|
||||
yield current_chunk
|
||||
except httpx.HTTPError as err:
|
||||
self.logger.error("HTTP error.", exc_info=err)
|
||||
yield "LLM service unavailable."
|
||||
except Exception as err:
|
||||
self.logger.error("Unexpected error.", exc_info=err)
|
||||
yield "Error processing the request."
|
||||
|
||||
async def _stream_query_llm(self, messages) -> AsyncGenerator[str]:
|
||||
"""Raises httpx.HTTPError when the API gives an error."""
|
||||
async with httpx.AsyncClient(timeout=None) as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
settings.llm_settings.local_llm_url,
|
||||
json={
|
||||
"model": settings.llm_settings.local_llm_model,
|
||||
"messages": messages,
|
||||
"temperature": 0.3,
|
||||
"stream": True,
|
||||
},
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
async for line in response.aiter_lines():
|
||||
if not line or not line.startswith("data: "):
|
||||
continue
|
||||
|
||||
data = line[len("data: ") :]
|
||||
if data.strip() == "[DONE]":
|
||||
break
|
||||
|
||||
try:
|
||||
event = json.loads(data)
|
||||
delta = event.get("choices", [{}])[0].get("delta", {}).get("content")
|
||||
if delta:
|
||||
yield delta
|
||||
except json.JSONDecodeError:
|
||||
self.logger.error("Failed to parse LLM response: %s", data)
|
||||
|
||||
@@ -3,10 +3,9 @@ import asyncio
|
||||
import numpy as np
|
||||
import zmq
|
||||
import zmq.asyncio as azmq
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
from spade.message import Message
|
||||
|
||||
from control_backend.agents import BaseAgent
|
||||
from control_backend.core.agent_system import InternalMessage
|
||||
from control_backend.core.config import settings
|
||||
|
||||
from .speech_recognizer import SpeechRecognizer
|
||||
@@ -19,53 +18,31 @@ class TranscriptionAgent(BaseAgent):
|
||||
"""
|
||||
|
||||
def __init__(self, audio_in_address: str):
|
||||
jid = settings.agent_settings.transcription_name + "@" + settings.agent_settings.host
|
||||
super().__init__(jid, settings.agent_settings.transcription_name)
|
||||
super().__init__(settings.agent_settings.transcription_name)
|
||||
|
||||
self.audio_in_address = audio_in_address
|
||||
self.audio_in_socket: azmq.Socket | None = None
|
||||
self.speech_recognizer = None
|
||||
self._concurrency = None
|
||||
|
||||
class TranscribingBehaviour(CyclicBehaviour):
|
||||
def __init__(self, audio_in_socket: azmq.Socket):
|
||||
super().__init__()
|
||||
max_concurrent_tasks = settings.behaviour_settings.transcription_max_concurrent_tasks
|
||||
self.audio_in_socket = audio_in_socket
|
||||
self.speech_recognizer = SpeechRecognizer.best_type()
|
||||
self._concurrency = asyncio.Semaphore(max_concurrent_tasks)
|
||||
async def setup(self):
|
||||
self.logger.info("Setting up %s", self.name)
|
||||
|
||||
def warmup(self):
|
||||
"""Load the transcription model into memory to speed up the first transcription."""
|
||||
self.speech_recognizer.load_model()
|
||||
self._connect_audio_in_socket()
|
||||
|
||||
async def _transcribe(self, audio: np.ndarray) -> str:
|
||||
async with self._concurrency:
|
||||
return await asyncio.to_thread(self.speech_recognizer.recognize_speech, audio)
|
||||
# Initialize recognizer and semaphore
|
||||
max_concurrent_tasks = settings.behaviour_settings.transcription_max_concurrent_tasks
|
||||
self._concurrency = asyncio.Semaphore(max_concurrent_tasks)
|
||||
self.speech_recognizer = SpeechRecognizer.best_type()
|
||||
self.speech_recognizer.load_model() # Warmup
|
||||
|
||||
async def _share_transcription(self, transcription: str):
|
||||
"""Share a transcription to the other agents that depend on it."""
|
||||
receiver_jids = [
|
||||
settings.agent_settings.text_belief_extractor_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host,
|
||||
] # Set message receivers here
|
||||
# Start background loop
|
||||
await self.add_background_task(self._transcribing_loop())
|
||||
|
||||
for receiver_jid in receiver_jids:
|
||||
message = Message(to=receiver_jid, body=transcription)
|
||||
await self.send(message)
|
||||
|
||||
async def run(self) -> None:
|
||||
audio = await self.audio_in_socket.recv()
|
||||
audio = np.frombuffer(audio, dtype=np.float32)
|
||||
speech = await self._transcribe(audio)
|
||||
if not speech:
|
||||
self.agent.logger.info("Nothing transcribed.")
|
||||
return
|
||||
|
||||
self.agent.logger.info("Transcribed speech: %s", speech)
|
||||
|
||||
await self._share_transcription(speech)
|
||||
self.logger.info("Finished setting up %s", self.name)
|
||||
|
||||
async def stop(self):
|
||||
assert self.audio_in_socket is not None
|
||||
self.audio_in_socket.close()
|
||||
self.audio_in_socket = None
|
||||
return await super().stop()
|
||||
@@ -75,13 +52,37 @@ class TranscriptionAgent(BaseAgent):
|
||||
self.audio_in_socket.setsockopt_string(zmq.SUBSCRIBE, "")
|
||||
self.audio_in_socket.connect(self.audio_in_address)
|
||||
|
||||
async def setup(self):
|
||||
self.logger.info("Setting up %s", self.jid)
|
||||
async def _transcribe(self, audio: np.ndarray) -> str:
|
||||
assert self._concurrency is not None and self.speech_recognizer is not None
|
||||
async with self._concurrency:
|
||||
return await asyncio.to_thread(self.speech_recognizer.recognize_speech, audio)
|
||||
|
||||
self._connect_audio_in_socket()
|
||||
async def _share_transcription(self, transcription: str):
|
||||
"""Share a transcription to the other agents that depend on it."""
|
||||
receiver_names = [
|
||||
settings.agent_settings.text_belief_extractor_name,
|
||||
]
|
||||
|
||||
transcribing = self.TranscribingBehaviour(self.audio_in_socket)
|
||||
transcribing.warmup()
|
||||
self.add_behaviour(transcribing)
|
||||
for receiver_name in receiver_names:
|
||||
message = InternalMessage(
|
||||
to=receiver_name,
|
||||
sender=self.name,
|
||||
body=transcription,
|
||||
)
|
||||
await self.send(message)
|
||||
|
||||
self.logger.info("Finished setting up %s", self.jid)
|
||||
async def _transcribing_loop(self) -> None:
|
||||
while self._running:
|
||||
try:
|
||||
assert self.audio_in_socket is not None
|
||||
audio_data = await self.audio_in_socket.recv()
|
||||
audio = np.frombuffer(audio_data, dtype=np.float32)
|
||||
speech = await self._transcribe(audio)
|
||||
if not speech:
|
||||
self.logger.info("Nothing transcribed.")
|
||||
continue
|
||||
|
||||
self.logger.info("Transcribed speech: %s", speech)
|
||||
await self._share_transcription(speech)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error in transcription loop: {e}")
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import asyncio
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import zmq
|
||||
import zmq.asyncio as azmq
|
||||
from spade.behaviour import CyclicBehaviour
|
||||
|
||||
from control_backend.agents import BaseAgent
|
||||
from control_backend.core.config import settings
|
||||
@@ -26,7 +27,7 @@ class SocketPoller[T]:
|
||||
:param timeout_ms: A timeout in milliseconds to wait for data.
|
||||
"""
|
||||
self.socket = socket
|
||||
self.poller = zmq.Poller()
|
||||
self.poller = azmq.Poller()
|
||||
self.poller.register(self.socket, zmq.POLLIN)
|
||||
self.timeout_ms = timeout_ms
|
||||
|
||||
@@ -38,81 +39,12 @@ class SocketPoller[T]:
|
||||
:return: Data from the socket or None.
|
||||
"""
|
||||
timeout_ms = timeout_ms or self.timeout_ms
|
||||
socks = dict(self.poller.poll(timeout_ms))
|
||||
socks = dict(await self.poller.poll(timeout_ms))
|
||||
if socks.get(self.socket) == zmq.POLLIN:
|
||||
return await self.socket.recv()
|
||||
return None
|
||||
|
||||
|
||||
class StreamingBehaviour(CyclicBehaviour):
|
||||
def __init__(self, audio_in_socket: azmq.Socket, audio_out_socket: azmq.Socket):
|
||||
super().__init__()
|
||||
self.audio_in_poller = SocketPoller[bytes](audio_in_socket)
|
||||
self.model, _ = torch.hub.load(
|
||||
repo_or_dir=settings.vad_settings.repo_or_dir,
|
||||
model=settings.vad_settings.model_name,
|
||||
force_reload=False,
|
||||
)
|
||||
self.audio_out_socket = audio_out_socket
|
||||
|
||||
self.audio_buffer = np.array([], dtype=np.float32)
|
||||
self.i_since_speech = (
|
||||
settings.behaviour_settings.vad_initial_since_speech
|
||||
) # Used to allow small pauses in speech
|
||||
self._ready = False
|
||||
|
||||
async def reset(self):
|
||||
"""Clears the ZeroMQ queue and tells this behavior to start."""
|
||||
discarded = 0
|
||||
# Poll for the shortest amount of time possible to clear the queue
|
||||
while await self.audio_in_poller.poll(1) is not None:
|
||||
discarded += 1
|
||||
self.agent.logger.info(f"Discarded {discarded} audio packets before starting.")
|
||||
self._ready = True
|
||||
|
||||
async def run(self) -> None:
|
||||
if not self._ready:
|
||||
return
|
||||
|
||||
data = await self.audio_in_poller.poll()
|
||||
if data is None:
|
||||
if len(self.audio_buffer) > 0:
|
||||
self.agent.logger.debug(
|
||||
"No audio data received. Discarding buffer until new data arrives."
|
||||
)
|
||||
self.audio_buffer = np.array([], dtype=np.float32)
|
||||
self.i_since_speech = settings.behaviour_settings.vad_initial_since_speech
|
||||
return
|
||||
|
||||
# copy otherwise Torch will be sad that it's immutable
|
||||
chunk = np.frombuffer(data, dtype=np.float32).copy()
|
||||
prob = self.model(torch.from_numpy(chunk), settings.vad_settings.sample_rate_hz).item()
|
||||
non_speech_patience = settings.behaviour_settings.vad_non_speech_patience_chunks
|
||||
prob_threshold = settings.behaviour_settings.vad_prob_threshold
|
||||
|
||||
if prob > prob_threshold:
|
||||
if self.i_since_speech > non_speech_patience:
|
||||
self.agent.logger.debug("Speech started.")
|
||||
self.audio_buffer = np.append(self.audio_buffer, chunk)
|
||||
self.i_since_speech = 0
|
||||
return
|
||||
self.i_since_speech += 1
|
||||
|
||||
# prob < 0.5, so speech maybe ended. Wait a bit more before to be more certain
|
||||
if self.i_since_speech <= non_speech_patience:
|
||||
self.audio_buffer = np.append(self.audio_buffer, chunk)
|
||||
return
|
||||
|
||||
# Speech probably ended. Make sure we have a usable amount of data.
|
||||
if len(self.audio_buffer) >= 3 * len(chunk):
|
||||
self.agent.logger.debug("Speech ended.")
|
||||
await self.audio_out_socket.send(self.audio_buffer[: -2 * len(chunk)].tobytes())
|
||||
|
||||
# At this point, we know that the speech has ended.
|
||||
# Prepend the last chunk that had no speech, for a more fluent boundary
|
||||
self.audio_buffer = chunk
|
||||
|
||||
|
||||
class VADAgent(BaseAgent):
|
||||
"""
|
||||
An agent which listens to an audio stream, does Voice Activity Detection (VAD), and sends
|
||||
@@ -120,16 +52,54 @@ class VADAgent(BaseAgent):
|
||||
"""
|
||||
|
||||
def __init__(self, audio_in_address: str, audio_in_bind: bool):
|
||||
jid = settings.agent_settings.vad_name + "@" + settings.agent_settings.host
|
||||
super().__init__(jid, settings.agent_settings.vad_name)
|
||||
super().__init__(settings.agent_settings.vad_name)
|
||||
|
||||
self.audio_in_address = audio_in_address
|
||||
self.audio_in_bind = audio_in_bind
|
||||
|
||||
self.audio_in_socket: azmq.Socket | None = None
|
||||
self.audio_out_socket: azmq.Socket | None = None
|
||||
self.audio_in_poller: SocketPoller | None = None
|
||||
|
||||
self.streaming_behaviour: StreamingBehaviour | None = None
|
||||
self.audio_buffer = np.array([], dtype=np.float32)
|
||||
self.i_since_speech = settings.behaviour_settings.vad_initial_since_speech
|
||||
self._ready = False
|
||||
self.model = None
|
||||
|
||||
async def setup(self):
|
||||
self.logger.info("Setting up %s", self.jid)
|
||||
|
||||
self._connect_audio_in_socket()
|
||||
|
||||
audio_out_port = self._connect_audio_out_socket()
|
||||
if audio_out_port is None:
|
||||
self.logger.error("Could not bind output socket, stopping.")
|
||||
await self.stop()
|
||||
return
|
||||
audio_out_address = f"tcp://localhost:{audio_out_port}"
|
||||
|
||||
# Initialize VAD model
|
||||
try:
|
||||
self.model, _ = torch.hub.load(
|
||||
repo_or_dir=settings.vad_settings.repo_or_dir,
|
||||
model=settings.vad_settings.model_name,
|
||||
force_reload=False,
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Failed to load VAD model.")
|
||||
await self.stop()
|
||||
return
|
||||
|
||||
# Warmup/reset
|
||||
await self.reset_stream()
|
||||
|
||||
await self.add_background_task(self._streaming_loop())
|
||||
|
||||
# Start agents dependent on the output audio fragments here
|
||||
transcriber = TranscriptionAgent(audio_out_address)
|
||||
await transcriber.start()
|
||||
|
||||
self.logger.info("Finished setting up %s", self.jid)
|
||||
|
||||
async def stop(self):
|
||||
"""
|
||||
@@ -141,7 +111,7 @@ class VADAgent(BaseAgent):
|
||||
if self.audio_out_socket is not None:
|
||||
self.audio_out_socket.close()
|
||||
self.audio_out_socket = None
|
||||
return await super().stop()
|
||||
await super().stop()
|
||||
|
||||
def _connect_audio_in_socket(self):
|
||||
self.audio_in_socket = azmq.Context.instance().socket(zmq.SUB)
|
||||
@@ -156,28 +126,67 @@ class VADAgent(BaseAgent):
|
||||
"""Returns the port bound, or None if binding failed."""
|
||||
try:
|
||||
self.audio_out_socket = azmq.Context.instance().socket(zmq.PUB)
|
||||
return self.audio_out_socket.bind_to_random_port("tcp://*", max_tries=100)
|
||||
return self.audio_out_socket.bind_to_random_port("tcp://localhost", max_tries=100)
|
||||
except zmq.ZMQBindError:
|
||||
self.logger.error("Failed to bind an audio output socket after 100 tries.")
|
||||
self.audio_out_socket = None
|
||||
return None
|
||||
|
||||
async def setup(self):
|
||||
self.logger.info("Setting up %s", self.jid)
|
||||
async def reset_stream(self):
|
||||
"""
|
||||
Clears the ZeroMQ queue and sets ready state.
|
||||
"""
|
||||
discarded = 0
|
||||
assert self.audio_in_poller is not None
|
||||
while await self.audio_in_poller.poll(1) is not None:
|
||||
discarded += 1
|
||||
self.logger.info(f"Discarded {discarded} audio packets before starting.")
|
||||
self._ready = True
|
||||
|
||||
self._connect_audio_in_socket()
|
||||
async def _streaming_loop(self):
|
||||
while self._running:
|
||||
if not self._ready:
|
||||
await asyncio.sleep(0.1)
|
||||
continue
|
||||
|
||||
audio_out_port = self._connect_audio_out_socket()
|
||||
if audio_out_port is None:
|
||||
await self.stop()
|
||||
return
|
||||
audio_out_address = f"tcp://localhost:{audio_out_port}"
|
||||
assert self.audio_in_poller is not None
|
||||
data = await self.audio_in_poller.poll()
|
||||
if data is None:
|
||||
if len(self.audio_buffer) > 0:
|
||||
self.logger.debug(
|
||||
"No audio data received. Discarding buffer until new data arrives."
|
||||
)
|
||||
self.audio_buffer = np.array([], dtype=np.float32)
|
||||
self.i_since_speech = settings.behaviour_settings.vad_initial_since_speech
|
||||
continue
|
||||
|
||||
self.streaming_behaviour = StreamingBehaviour(self.audio_in_socket, self.audio_out_socket)
|
||||
self.add_behaviour(self.streaming_behaviour)
|
||||
# copy otherwise Torch will be sad that it's immutable
|
||||
chunk = np.frombuffer(data, dtype=np.float32).copy()
|
||||
assert self.model is not None
|
||||
prob = self.model(torch.from_numpy(chunk), settings.vad_settings.sample_rate_hz).item()
|
||||
non_speech_patience = settings.behaviour_settings.vad_non_speech_patience_chunks
|
||||
prob_threshold = settings.behaviour_settings.vad_prob_threshold
|
||||
|
||||
# Start agents dependent on the output audio fragments here
|
||||
transcriber = TranscriptionAgent(audio_out_address)
|
||||
await transcriber.start()
|
||||
if prob > prob_threshold:
|
||||
if self.i_since_speech > non_speech_patience:
|
||||
self.logger.debug("Speech started.")
|
||||
self.audio_buffer = np.append(self.audio_buffer, chunk)
|
||||
self.i_since_speech = 0
|
||||
continue
|
||||
|
||||
self.logger.info("Finished setting up %s", self.jid)
|
||||
self.i_since_speech += 1
|
||||
|
||||
# prob < threshold, so speech maybe ended. Wait a bit more before to be more certain
|
||||
if self.i_since_speech <= non_speech_patience:
|
||||
self.audio_buffer = np.append(self.audio_buffer, chunk)
|
||||
continue
|
||||
|
||||
# Speech probably ended. Make sure we have a usable amount of data.
|
||||
if len(self.audio_buffer) >= 3 * len(chunk):
|
||||
self.logger.debug("Speech ended.")
|
||||
assert self.audio_out_socket is not None
|
||||
await self.audio_out_socket.send(self.audio_buffer[: -2 * len(chunk)].tobytes())
|
||||
|
||||
# At this point, we know that the speech has ended.
|
||||
# Prepend the last chunk that had no speech, for a more fluent boundary
|
||||
self.audio_buffer = chunk
|
||||
|
||||
Reference in New Issue
Block a user