pepperplus/pepperplus-cb
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into demo

Browse Source 
# Conflicts:
#	src/control_backend/agents/bdi/behaviours/receive_llm_resp_behaviour.py
#	src/control_backend/agents/llm/llm.py
#	src/control_backend/agents/ri_command_agent.py
#	src/control_backend/agents/transcription/speech_recognizer.py
This commit is contained in:
Twirre Meulenbelt
2025-11-02 21:07:50 +01:00
parent 4ffe3b2071 c6edad0bb4
commit 8e4d8f9d1e
32 changed files with 412 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/control_backend/agents/bdi/bdi_core.py
View File

@@ -58,11 +58,11 @@ class BDICoreAgent(BDIAgent):
class SendBehaviour(OneShotBehaviour):
async def run(self) -> None:
msg = Message(
to= settings.agent_settings.llm_agent_name + '@' + settings.agent_settings.host,
body= text
to=settings.agent_settings.llm_agent_name + "@" + settings.agent_settings.host,
body=text,
)
await self.send(msg)
self.agent.logger.info("Message sent to LLM: %s", text)
self.add_behaviour(SendBehaviour())
self.add_behaviour(SendBehaviour())

6
src/control_backend/agents/bdi/behaviours/belief_setter.py
View File

@@ -3,7 +3,7 @@ import logging
from spade.agent import Message
from spade.behaviour import CyclicBehaviour
from spade_bdi.bdi import BDIAgent, BeliefNotInitiated
from spade_bdi.bdi import BDIAgent
from control_backend.core.config import settings
@@ -23,7 +23,6 @@ class BeliefSetterBehaviour(CyclicBehaviour):
self.logger.info(f"Received message {msg.body}")
self._process_message(msg)
def _process_message(self, message: Message):
sender = message.sender.node # removes host from jid and converts to str
self.logger.debug("Sender: %s", sender)
@@ -61,6 +60,7 @@ class BeliefSetterBehaviour(CyclicBehaviour):
self.agent.bdi.set_belief(belief, *arguments)
# Special case: if there's a new user message, flag that we haven't responded yet
if belief == "user_said": self.agent.bdi.set_belief("new_message")
if belief == "user_said":
self.agent.bdi.set_belief("new_message")
self.logger.info("Set belief %s with arguments %s", belief, arguments)

16
src/control_backend/agents/bdi/behaviours/receive_llm_resp_behaviour.py
View File

@@ -11,6 +11,7 @@ class ReceiveLLMResponseBehaviour(CyclicBehaviour):
"""
Adds behavior to receive responses from the LLM Agent.
"""
logger = logging.getLogger("BDI/LLM Receiver")
async def run(self):
@@ -18,7 +19,7 @@ class ReceiveLLMResponseBehaviour(CyclicBehaviour):
if not msg:
return
sender = msg.sender.node
sender = msg.sender.node
match sender:
case settings.agent_settings.llm_agent_name:
content = msg.body
@@ -26,14 +27,17 @@ class ReceiveLLMResponseBehaviour(CyclicBehaviour):
speech_command = SpeechCommand(data=content)
message = Message(to=settings.agent_settings.ri_command_agent_name
+ '@' + settings.agent_settings.host,
sender=self.agent.jid,
body=speech_command.model_dump_json())
message = Message(
to=settings.agent_settings.ri_command_agent_name
+ "@"
+ settings.agent_settings.host,
sender=self.agent.jid,
body=speech_command.model_dump_json(),
)
self.logger.debug("Sending message: %s", message)
await self.send(message)
case _:
self.logger.debug("Not from the llm, discarding message")
pass
pass

40
src/control_backend/agents/bdi/behaviours/text_belief_extractor.py
View File

@@ -13,28 +13,30 @@ class BeliefFromText(CyclicBehaviour):
# TODO: LLM prompt nog hardcoded
llm_instruction_prompt = """
You are an information extraction assistent for a BDI agent. Your task is to extract values from a user's text to bind a list of ungrounded beliefs. Rules:
You will receive a JSON object with "beliefs" (a list of ungrounded AgentSpeak beliefs) and "text" (user's transcript).
You are an information extraction assistent for a BDI agent. Your task is to extract values \
from a user's text to bind a list of ungrounded beliefs. Rules:
You will receive a JSON object with "beliefs" (a list of ungrounded AgentSpeak beliefs) \
and "text" (user's transcript).
Analyze the text to find values that sematically match the variables (X,Y,Z) in the beliefs.
A single piece of text might contain multiple instances that match a belief.
Respond ONLY with a single JSON object.
The JSON object's keys should be the belief functors (e.g., "weather").
The value for each key must be a list of lists.
Each inner list must contain the extracted arguments (as strings) for one instance of that belief.
CRITICAL: If no information in the text matches a belief, DO NOT include that key in your response.
Each inner list must contain the extracted arguments (as strings) for one instance \
of that belief.
CRITICAL: If no information in the text matches a belief, DO NOT include that key \
in your response.
"""
# on_start agent receives message containing the beliefs to look out for and sets up the LLM with instruction prompt
#async def on_start(self):
# on_start agent receives message containing the beliefs to look out for and
# sets up the LLM with instruction prompt
# async def on_start(self):
# msg = await self.receive(timeout=0.1)
# self.beliefs = dict uit message
# send instruction prompt to LLM
beliefs: dict[str, list[str]]
beliefs = {
"mood": ["X"],
"car": ["Y"]
}
beliefs = {"mood": ["X"], "car": ["Y"]}
async def run(self):
msg = await self.receive(timeout=0.1)
@@ -58,8 +60,8 @@ class BeliefFromText(CyclicBehaviour):
prompt = text_prompt + beliefs_prompt
self.logger.info(prompt)
#prompt_msg = Message(to="LLMAgent@whatever")
#response = self.send(prompt_msg)
# prompt_msg = Message(to="LLMAgent@whatever")
# response = self.send(prompt_msg)
# Mock response; response is beliefs in JSON format, it parses do dict[str,list[list[str]]]
response = '{"mood": [["happy"]]}'
@@ -67,8 +69,9 @@ class BeliefFromText(CyclicBehaviour):
try:
json.loads(response)
belief_message = Message(
to=settings.agent_settings.bdi_core_agent_name + '@' + settings.agent_settings.host,
body=response)
to=settings.agent_settings.bdi_core_agent_name + "@" + settings.agent_settings.host,
body=response,
)
belief_message.thread = "beliefs"
await self.send(belief_message)
@@ -85,9 +88,12 @@ class BeliefFromText(CyclicBehaviour):
"""
belief = {"beliefs": {"user_said": [txt]}, "type": "belief_extraction_text"}
payload = json.dumps(belief)
belief_msg = Message(to=settings.agent_settings.belief_collector_agent_name
+ '@' + settings.agent_settings.host,
body=payload)
belief_msg = Message(
to=settings.agent_settings.belief_collector_agent_name
+ "@"
+ settings.agent_settings.host,
body=payload,
)
belief_msg.thread = "beliefs"
await self.send(belief_msg)

2
src/control_backend/agents/bdi/text_extractor.py
View File

@@ -6,4 +6,4 @@ from control_backend.agents.bdi.behaviours.text_belief_extractor import BeliefFr
class TBeliefExtractor(Agent):
async def setup(self):
self.b = BeliefFromText()
self.add_behaviour(self.b)
self.add_behaviour(self.b)

38
src/control_backend/agents/belief_collector/behaviours/continuous_collect.py
View File

@@ -1,11 +1,14 @@
import json
import logging
from spade.behaviour import CyclicBehaviour
from spade.agent import Message
from spade.behaviour import CyclicBehaviour
from control_backend.core.config import settings
logger = logging.getLogger(__name__)
class ContinuousBeliefCollector(CyclicBehaviour):
"""
Continuously collects beliefs/emotions from extractor agents:
@@ -17,7 +20,6 @@ class ContinuousBeliefCollector(CyclicBehaviour):
if msg:
await self._process_message(msg)
async def _process_message(self, msg: Message):
sender_node = self._sender_node(msg)
@@ -27,7 +29,9 @@ class ContinuousBeliefCollector(CyclicBehaviour):
except Exception as e:
logger.warning(
"BeliefCollector: failed to parse JSON from %s. Body=%r Error=%s",
sender_node, msg.body, e
sender_node,
msg.body,
e,
)
return
@@ -35,16 +39,21 @@ class ContinuousBeliefCollector(CyclicBehaviour):
# Prefer explicit 'type' field
if msg_type == "belief_extraction_text" or sender_node == "belief_text_agent_mock":
logger.info("BeliefCollector: message routed to _handle_belief_text (sender=%s)", sender_node)
logger.info(
"BeliefCollector: message routed to _handle_belief_text (sender=%s)", sender_node
)
await self._handle_belief_text(payload, sender_node)
#This is not implemented yet, but we keep the structure for future use
elif msg_type == "emotion_extraction_text" or sender_node == "emo_text_agent_mock":
logger.info("BeliefCollector: message routed to _handle_emo_text (sender=%s)", sender_node)
# This is not implemented yet, but we keep the structure for future use
elif msg_type == "emotion_extraction_text" or sender_node == "emo_text_agent_mock":
logger.info(
"BeliefCollector: message routed to _handle_emo_text (sender=%s)", sender_node
)
await self._handle_emo_text(payload, sender_node)
else:
logger.info(
"BeliefCollector: unrecognized message (sender=%s, type=%r). Ignoring.",
sender_node, msg_type
sender_node,
msg_type,
)
@staticmethod
@@ -56,13 +65,12 @@ class ContinuousBeliefCollector(CyclicBehaviour):
s = str(msg.sender) if msg.sender is not None else "no_sender"
return s.split("@", 1)[0] if "@" in s else s
async def _handle_belief_text(self, payload: dict, origin: str):
"""
Expected payload:
{
"type": "belief_extraction_text",
"beliefs": {"user_said": ["hello"","Can you help me?","stop talking to me","No","Pepper do a dance"]}
"beliefs": {"user_said": ["Can you help me?"]}
}
@@ -72,11 +80,11 @@ class ContinuousBeliefCollector(CyclicBehaviour):
if not beliefs:
logger.info("BeliefCollector: no beliefs to process.")
return
if not isinstance(beliefs, dict):
logger.warning("BeliefCollector: 'beliefs' is not a dict: %r", beliefs)
return
if not all(isinstance(v, list) for v in beliefs.values()):
logger.warning("BeliefCollector: 'beliefs' values are not all lists: %r", beliefs)
return
@@ -84,17 +92,14 @@ class ContinuousBeliefCollector(CyclicBehaviour):
logger.info("BeliefCollector: forwarding %d beliefs.", len(beliefs))
for belief_name, belief_list in beliefs.items():
for belief in belief_list:
logger.info(" - %s %s", belief_name,str(belief))
logger.info(" - %s %s", belief_name, str(belief))
await self._send_beliefs_to_bdi(beliefs, origin=origin)
async def _handle_emo_text(self, payload: dict, origin: str):
"""TODO: implement (after we have emotional recogntion)"""
pass
async def _send_beliefs_to_bdi(self, beliefs: list[str], origin: str | None = None):
"""
Sends a unified belief packet to the BDI agent.
@@ -107,6 +112,5 @@ class ContinuousBeliefCollector(CyclicBehaviour):
msg = Message(to=to_jid, sender=self.agent.jid, thread="beliefs")
msg.body = json.dumps(beliefs)
await self.send(msg)
logger.info("BeliefCollector: sent %d belief(s) to BDI at %s", len(beliefs), to_jid)

4
src/control_backend/agents/belief_collector/belief_collector.py
View File

@@ -1,13 +1,15 @@
import logging
from spade.agent import Agent
from .behaviours.continuous_collect import ContinuousBeliefCollector
logger = logging.getLogger(__name__)
class BeliefCollectorAgent(Agent):
async def setup(self):
logger.info("BeliefCollectorAgent starting (%s)", self.jid)
# Attach the continuous collector behaviour (listens and forwards to BDI)
self.add_behaviour(ContinuousBeliefCollector())
logger.info("BeliefCollectorAgent ready.")
logger.info("BeliefCollectorAgent ready.")

31
src/control_backend/agents/llm/llm.py
View File

@@ -2,10 +2,11 @@
LLM Agent module for routing text queries from the BDI Core Agent to a local LLM
service and returning its responses back to the BDI Core Agent.
"""
import json
import logging
import re
from typing import AsyncGenerator
from collections.abc import AsyncGenerator
import httpx
from spade.agent import Agent
@@ -62,16 +63,17 @@ class LLMAgent(Agent):
# Consume the streaming generator and send a reply for every chunk
async for chunk in self._query_llm(user_text):
await self._reply(chunk)
self.agent.logger.debug("Finished processing BDI message. "
"Response sent in chunks to BDI Core Agent.")
self.agent.logger.debug(
"Finished processing BDI message. Response sent in chunks to BDI Core Agent."
)
async def _reply(self, msg: str):
"""
Sends a response message back to the BDI Core Agent.
"""
reply = Message(
to=settings.agent_settings.bdi_core_agent_name + '@' + settings.agent_settings.host,
body=msg
to=settings.agent_settings.bdi_core_agent_name + "@" + settings.agent_settings.host,
body=msg,
)
await self.send(reply)
@@ -100,7 +102,7 @@ class LLMAgent(Agent):
{
"role": "user",
"content": prompt,
}
},
]
try:
@@ -111,8 +113,7 @@ class LLMAgent(Agent):
# Stream the message in chunks separated by punctuation.
# We include the delimiter in the emitted chunk for natural flow.
pattern = re.compile(
r".*?(?:,|;|:|—|–|-|\.{3}|…|\.|\?|!|\(|\)|\[|\]|/)\s*",
re.DOTALL
r".*?(?:,|;|:|—|–|-|\.{3}|…|\.|\?|!|\(|\)|\[|\]|/)\s*", re.DOTALL
)
for m in pattern.finditer(current_chunk):
chunk = m.group(0)
@@ -121,7 +122,8 @@ class LLMAgent(Agent):
current_chunk = ""
# Yield any remaining tail
if current_chunk: yield current_chunk
if current_chunk:
yield current_chunk
except httpx.HTTPError as err:
self.agent.logger.error("HTTP error.", exc_info=err)
yield "LLM service unavailable."
@@ -145,15 +147,18 @@ class LLMAgent(Agent):
response.raise_for_status()
async for line in response.aiter_lines():
if not line or not line.startswith("data: "): continue
if not line or not line.startswith("data: "):
continue
data = line[len("data: "):]
if data.strip() == "[DONE]": break
data = line[len("data: ") :]
if data.strip() == "[DONE]":
break
try:
event = json.loads(data)
delta = event.get("choices", [{}])[0].get("delta", {}).get("content")
if delta: yield delta
if delta:
yield delta
except json.JSONDecodeError:
self.agent.logger.error("Failed to parse LLM response: %s", data)

19
src/control_backend/agents/mock_agents/belief_text_mock.py
View File

@@ -1,18 +1,33 @@
import json
from spade.agent import Agent
from spade.behaviour import OneShotBehaviour
from spade.message import Message
from control_backend.core.config import settings
class BeliefTextAgent(Agent):
class SendOnceBehaviourBlfText(OneShotBehaviour):
async def run(self):
to_jid = f"{settings.agent_settings.belief_collector_agent_name}@{settings.agent_settings.host}"
to_jid = (
settings.agent_settings.belief_collector_agent_name
+ "@"
+ settings.agent_settings.host
)
# Send multiple beliefs in one JSON payload
payload = {
"type": "belief_extraction_text",
"beliefs": {"user_said": ["hello test","Can you help me?","stop talking to me","No","Pepper do a dance"]}
"beliefs": {
"user_said": [
"hello test",
"Can you help me?",
"stop talking to me",
"No",
"Pepper do a dance",
]
},
}
msg = Message(to=to_jid)

4
src/control_backend/agents/ri_command_agent.py
View File

@@ -2,9 +2,9 @@ import json
import logging
import spade.agent
import zmq
from spade.agent import Agent
from spade.behaviour import CyclicBehaviour
import zmq
from control_backend.core.config import settings
from control_backend.core.zmq_context import context
@@ -34,6 +34,7 @@ class RICommandAgent(Agent):
class SendCommandsBehaviour(CyclicBehaviour):
"""Behaviour for sending commands received from the UI."""
async def run(self):
"""
Run the command publishing loop indefinetely.
@@ -54,6 +55,7 @@ class RICommandAgent(Agent):
class SendPythonCommandsBehaviour(CyclicBehaviour):
"""Behaviour for sending commands received from other Python agents."""
async def run(self):
message: spade.agent.Message = await self.receive(timeout=0.1)
if message and message.to == self.agent.jid:

11
src/control_backend/agents/ri_communication_agent.py
View File

@@ -1,14 +1,13 @@
import asyncio
import json
import logging
import zmq
from spade.agent import Agent
from spade.behaviour import CyclicBehaviour
import zmq
from control_backend.agents.ri_command_agent import RICommandAgent
from control_backend.core.config import settings
from control_backend.core.zmq_context import context
from control_backend.schemas.message import Message
from control_backend.agents.ri_command_agent import RICommandAgent
logger = logging.getLogger(__name__)
@@ -47,7 +46,7 @@ class RICommunicationAgent(Agent):
message = await asyncio.wait_for(self.agent.req_socket.recv_json(), timeout=3.0)
# We didnt get a reply :(
except asyncio.TimeoutError as e:
except TimeoutError:
logger.info("No ping retrieved in 3 seconds, killing myself.")
self.kill()
@@ -88,7 +87,7 @@ class RICommunicationAgent(Agent):
try:
received_message = await asyncio.wait_for(self.req_socket.recv_json(), timeout=20.0)
except asyncio.TimeoutError:
except TimeoutError:
logger.warning(
"No connection established in 20 seconds (attempt %d/%d)",
retries + 1,

14
src/control_backend/agents/transcription/speech_recognizer.py
View File

@@ -75,7 +75,8 @@ class MLXWhisperSpeechRecognizer(SpeechRecognizer):
self.model_name = "mlx-community/whisper-small.en-mlx"
def load_model(self):
if self.was_loaded: return
if self.was_loaded:
return
# There appears to be no dedicated mechanism to preload a model, but this `get_model` does
# store it in memory for later usage
ModelHolder.get_model(self.model_name, mx.float16)
@@ -83,7 +84,7 @@ class MLXWhisperSpeechRecognizer(SpeechRecognizer):
def recognize_speech(self, audio: np.ndarray) -> str:
self.load_model()
return mlx_whisper.transcribe(audio, path_or_hf_repo=self.model_name)["text"].strip()
return mlx_whisper.transcribe(audio, path_or_hf_repo=self.model_name)["text"]
class OpenAIWhisperSpeechRecognizer(SpeechRecognizer):
@@ -92,12 +93,13 @@ class OpenAIWhisperSpeechRecognizer(SpeechRecognizer):
self.model = None
def load_model(self):
if self.model is not None: return
if self.model is not None:
return
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
self.model = whisper.load_model("small.en", device=device)
def recognize_speech(self, audio: np.ndarray) -> str:
self.load_model()
return whisper.transcribe(self.model,
audio,
decode_options=self._get_decode_options(audio))["text"]
return whisper.transcribe(
self.model, audio, decode_options=self._get_decode_options(audio)
)["text"]

3
src/control_backend/agents/transcription/transcription_agent.py
View File

@@ -47,7 +47,8 @@ class TranscriptionAgent(Agent):
"""Share a transcription to the other agents that depend on it."""
receiver_jids = [
settings.agent_settings.text_belief_extractor_agent_name
+ '@' + settings.agent_settings.host,
+ "@"
+ settings.agent_settings.host,
] # Set message receivers here
for receiver_jid in receiver_jids:

5
src/control_backend/api/v1/endpoints/command.py
View File

@@ -1,9 +1,9 @@
from fastapi import APIRouter, Request
import logging
from fastapi import APIRouter, Request
from zmq import Socket
from control_backend.schemas.ri_message import SpeechCommand, RIEndpoint
from control_backend.schemas.ri_message import SpeechCommand
logger = logging.getLogger(__name__)
@@ -17,6 +17,5 @@ async def receive_command(command: SpeechCommand, request: Request):
topic = b"command"
pub_socket: Socket = request.app.state.internal_comm_socket
pub_socket.send_multipart([topic, command.model_dump_json().encode()])
return {"status": "Command received"}

2
src/control_backend/api/v1/router.py
View File

@@ -1,6 +1,6 @@
from fastapi.routing import APIRouter
from control_backend.api.v1.endpoints import message, sse, command
from control_backend.api.v1.endpoints import command, message, sse
api_router = APIRouter()

2
src/control_backend/core/config.py
View File

@@ -24,6 +24,7 @@ class LLMSettings(BaseModel):
local_llm_url: str = "http://localhost:1234/v1/chat/completions"
local_llm_model: str = "openai/gpt-oss-20b"
class Settings(BaseSettings):
app_title: str = "PepperPlus"
@@ -37,4 +38,5 @@ class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env")
settings = Settings()

20
src/control_backend/main.py
View File

@@ -8,13 +8,14 @@ import zmq
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
# Internal imports
from control_backend.agents.ri_communication_agent import RICommunicationAgent
from control_backend.agents.bdi.bdi_core import BDICoreAgent
from control_backend.agents.vad_agent import VADAgent
from control_backend.agents.llm.llm import LLMAgent
from control_backend.agents.bdi.text_extractor import TBeliefExtractor
from control_backend.agents.belief_collector.belief_collector import BeliefCollectorAgent
from control_backend.agents.llm.llm import LLMAgent
# Internal imports
from control_backend.agents.ri_communication_agent import RICommunicationAgent
from control_backend.agents.vad_agent import VADAgent
from control_backend.api.v1.router import api_router
from control_backend.core.config import settings
from control_backend.core.zmq_context import context
@@ -34,7 +35,6 @@ async def lifespan(app: FastAPI):
app.state.internal_comm_socket = internal_comm_socket
logger.info("Internal publishing socket bound to %s", internal_comm_socket)
# Initiate agents
ri_communication_agent = RICommunicationAgent(
settings.agent_settings.ri_communication_agent_name + "@" + settings.agent_settings.host,
@@ -45,26 +45,28 @@ async def lifespan(app: FastAPI):
await ri_communication_agent.start()
llm_agent = LLMAgent(
settings.agent_settings.llm_agent_name + '@' + settings.agent_settings.host,
settings.agent_settings.llm_agent_name + "@" + settings.agent_settings.host,
settings.agent_settings.llm_agent_name,
)
await llm_agent.start()
bdi_core = BDICoreAgent(
settings.agent_settings.bdi_core_agent_name + '@' + settings.agent_settings.host,
settings.agent_settings.bdi_core_agent_name + "@" + settings.agent_settings.host,
settings.agent_settings.bdi_core_agent_name,
"src/control_backend/agents/bdi/rules.asl",
)
await bdi_core.start()
belief_collector = BeliefCollectorAgent(
settings.agent_settings.belief_collector_agent_name + '@' + settings.agent_settings.host,
settings.agent_settings.belief_collector_agent_name + "@" + settings.agent_settings.host,
settings.agent_settings.belief_collector_agent_name,
)
await belief_collector.start()
text_belief_extractor = TBeliefExtractor(
settings.agent_settings.text_belief_extractor_agent_name + '@' + settings.agent_settings.host,
settings.agent_settings.text_belief_extractor_agent_name
+ "@"
+ settings.agent_settings.host,
settings.agent_settings.text_belief_extractor_agent_name,
)
await text_belief_extractor.start()

4
src/control_backend/schemas/ri_message.py
View File

@@ -1,7 +1,7 @@
from enum import Enum
from typing import Any, Literal
from typing import Any
from pydantic import BaseModel, Field, ValidationError
from pydantic import BaseModel
class RIEndpoint(str, Enum):