Merge branch 'dev' into refactor/logging
This commit is contained in:
@@ -9,18 +9,23 @@ from control_backend.core.config import settings
|
||||
class BeliefFromText(CyclicBehaviour):
|
||||
# TODO: LLM prompt nog hardcoded
|
||||
llm_instruction_prompt = """
|
||||
You are an information extraction assistent for a BDI agent. Your task is to extract values from a user's text to bind a list of ungrounded beliefs. Rules:
|
||||
You will receive a JSON object with "beliefs" (a list of ungrounded AgentSpeak beliefs) and "text" (user's transcript).
|
||||
You are an information extraction assistent for a BDI agent. Your task is to extract values \
|
||||
from a user's text to bind a list of ungrounded beliefs. Rules:
|
||||
You will receive a JSON object with "beliefs" (a list of ungrounded AgentSpeak beliefs) \
|
||||
and "text" (user's transcript).
|
||||
Analyze the text to find values that sematically match the variables (X,Y,Z) in the beliefs.
|
||||
A single piece of text might contain multiple instances that match a belief.
|
||||
Respond ONLY with a single JSON object.
|
||||
The JSON object's keys should be the belief functors (e.g., "weather").
|
||||
The value for each key must be a list of lists.
|
||||
Each inner list must contain the extracted arguments (as strings) for one instance of that belief.
|
||||
CRITICAL: If no information in the text matches a belief, DO NOT include that key in your response.
|
||||
Each inner list must contain the extracted arguments (as strings) for one instance \
|
||||
of that belief.
|
||||
CRITICAL: If no information in the text matches a belief, DO NOT include that key \
|
||||
in your response.
|
||||
"""
|
||||
|
||||
# on_start agent receives message containing the beliefs to look out for and sets up the LLM with instruction prompt
|
||||
# on_start agent receives message containing the beliefs to look out for and
|
||||
# sets up the LLM with instruction prompt
|
||||
# async def on_start(self):
|
||||
# msg = await self.receive(timeout=0.1)
|
||||
# self.beliefs = dict uit message
|
||||
|
||||
@@ -7,6 +7,7 @@ from spade.behaviour import CyclicBehaviour
|
||||
from control_backend.core.config import settings
|
||||
|
||||
|
||||
|
||||
class ContinuousBeliefCollector(CyclicBehaviour):
|
||||
"""
|
||||
Continuously collects beliefs/emotions from extractor agents:
|
||||
@@ -23,9 +24,12 @@ class ContinuousBeliefCollector(CyclicBehaviour):
|
||||
# Parse JSON payload
|
||||
try:
|
||||
payload = json.loads(msg.body)
|
||||
except JSONDecodeError as e:
|
||||
self.agent.logger.warning(
|
||||
"Failed to parse JSON from %s. Body=%r Error=%s", sender_node, msg.body, e
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"BeliefCollector: failed to parse JSON from %s. Body=%r Error=%s",
|
||||
sender_node,
|
||||
msg.body,
|
||||
e,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -51,7 +55,7 @@ class ContinuousBeliefCollector(CyclicBehaviour):
|
||||
Expected payload:
|
||||
{
|
||||
"type": "belief_extraction_text",
|
||||
"beliefs": {"user_said": ["hello"","Can you help me?","stop talking to me","No","Pepper do a dance"]}
|
||||
"beliefs": {"user_said": ["Can you help me?"]}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,11 @@ from control_backend.core.config import settings
|
||||
class BeliefTextAgent(Agent):
|
||||
class SendOnceBehaviourBlfText(OneShotBehaviour):
|
||||
async def run(self):
|
||||
to_jid = f"{settings.agent_settings.belief_collector_agent_name}@{settings.agent_settings.host}"
|
||||
to_jid = (
|
||||
settings.agent_settings.belief_collector_agent_name
|
||||
+ "@"
|
||||
+ settings.agent_settings.host
|
||||
)
|
||||
|
||||
# Send multiple beliefs in one JSON payload
|
||||
payload = {
|
||||
|
||||
@@ -75,7 +75,8 @@ class MLXWhisperSpeechRecognizer(SpeechRecognizer):
|
||||
self.model_name = "mlx-community/whisper-small.en-mlx"
|
||||
|
||||
def load_model(self):
|
||||
if self.was_loaded: return
|
||||
if self.was_loaded:
|
||||
return
|
||||
# There appears to be no dedicated mechanism to preload a model, but this `get_model` does
|
||||
# store it in memory for later usage
|
||||
ModelHolder.get_model(self.model_name, mx.float16)
|
||||
@@ -83,9 +84,9 @@ class MLXWhisperSpeechRecognizer(SpeechRecognizer):
|
||||
|
||||
def recognize_speech(self, audio: np.ndarray) -> str:
|
||||
self.load_model()
|
||||
return mlx_whisper.transcribe(audio,
|
||||
path_or_hf_repo=self.model_name,
|
||||
decode_options=self._get_decode_options(audio))["text"]
|
||||
return mlx_whisper.transcribe(
|
||||
audio, path_or_hf_repo=self.model_name, decode_options=self._get_decode_options(audio)
|
||||
)["text"]
|
||||
return mlx_whisper.transcribe(audio, path_or_hf_repo=self.model_name)["text"].strip()
|
||||
|
||||
|
||||
@@ -95,12 +96,13 @@ class OpenAIWhisperSpeechRecognizer(SpeechRecognizer):
|
||||
self.model = None
|
||||
|
||||
def load_model(self):
|
||||
if self.model is not None: return
|
||||
if self.model is not None:
|
||||
return
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
self.model = whisper.load_model("small.en", device=device)
|
||||
|
||||
def recognize_speech(self, audio: np.ndarray) -> str:
|
||||
self.load_model()
|
||||
return whisper.transcribe(self.model,
|
||||
audio,
|
||||
decode_options=self._get_decode_options(audio))["text"]
|
||||
return whisper.transcribe(
|
||||
self.model, audio, decode_options=self._get_decode_options(audio)
|
||||
)["text"]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from fastapi import APIRouter, Request
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from zmq import Socket
|
||||
|
||||
from control_backend.schemas.ri_message import SpeechCommand, RIEndpoint
|
||||
from control_backend.schemas.ri_message import SpeechCommand
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -17,6 +17,5 @@ async def receive_command(command: SpeechCommand, request: Request):
|
||||
topic = b"command"
|
||||
pub_socket: Socket = request.app.state.internal_comm_socket
|
||||
pub_socket.send_multipart([topic, command.model_dump_json().encode()])
|
||||
|
||||
|
||||
return {"status": "Command received"}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from fastapi.routing import APIRouter
|
||||
|
||||
from control_backend.api.v1.endpoints import message, sse, command
|
||||
from control_backend.api.v1.endpoints import command, message, sse
|
||||
|
||||
api_router = APIRouter()
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ class LLMSettings(BaseModel):
|
||||
local_llm_url: str = "http://localhost:1234/v1/chat/completions"
|
||||
local_llm_model: str = "openai/gpt-oss-20b"
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
app_title: str = "PepperPlus"
|
||||
|
||||
@@ -37,4 +38,5 @@ class Settings(BaseSettings):
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from enum import Enum
|
||||
from typing import Any, Literal
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class RIEndpoint(str, Enum):
|
||||
|
||||
Reference in New Issue
Block a user