feat: face recognition agent #53

Open
2584433 wants to merge 31 commits from feat/face-recognition into main
4 changed files with 71 additions and 17 deletions
Showing only changes of commit f6477e5325 - Show all commits

View File

@@ -23,6 +23,7 @@ from control_backend.schemas.program import (
BasicNorm, BasicNorm,
ConditionalNorm, ConditionalNorm,
EmotionBelief, EmotionBelief,
FaceBelief,
GestureAction, GestureAction,
Goal, Goal,
InferredBelief, InferredBelief,
@@ -465,6 +466,10 @@ class AgentSpeakGenerator:
def _(self, eb: EmotionBelief) -> AstExpression: def _(self, eb: EmotionBelief) -> AstExpression:
return AstLiteral("emotion_detected", [AstAtom(eb.emotion)]) return AstLiteral("emotion_detected", [AstAtom(eb.emotion)])
@_astify.register
def _(self, eb: FaceBelief) -> AstExpression:
return AstLiteral("face_present")
@_astify.register @_astify.register
def _(self, ib: InferredBelief) -> AstExpression: def _(self, ib: InferredBelief) -> AstExpression:
return AstBinaryOp( return AstBinaryOp(

View File

@@ -1,10 +1,14 @@
import asyncio import asyncio
from random import random
import zmq import zmq
import zmq.asyncio as azmq import zmq.asyncio as azmq
from zmq.asyncio import Context from zmq.asyncio import Context
from control_backend.agents import BaseAgent from control_backend.agents import BaseAgent
from control_backend.core.agent_system import InternalMessage
from control_backend.core.config import settings
from control_backend.schemas.belief_message import Belief, BeliefMessage
class FacePerceptionAgent(BaseAgent): class FacePerceptionAgent(BaseAgent):
@@ -51,19 +55,49 @@ class FacePerceptionAgent(BaseAgent):
if face_present != self._last_face_state: if face_present != self._last_face_state:
self._last_face_state = face_present self._last_face_state = face_present
self.logger.info("👀 Face detected" if face_present else "🙈 Face lost") self.logger.debug("Face detected" if face_present else "Face lost")
# TODO: post belief to BDI here await self._update_face_belief(face_present)
except Exception as e: except Exception as e:
self.logger.warning("Face polling failed") self.logger.warning("Face polling failed")
self.logger.warn(e) self.logger.warn(e)
i = random()
await self._update_face_belief(i > 0.5)
await asyncio.sleep(poll_interval) await asyncio.sleep(poll_interval)
async def _handle_face_change(self, present: bool): async def _post_face_belief(self, present: bool):
"""
Send a face_present belief update to the BDI Core Agent.
"""
if present: if present:
self.logger.info("👀 Face detected") belief_msg = BeliefMessage(create=[{"name": "face_present", "arguments": []}])
# await self.post_belief("face_present", value=True)
else: else:
self.logger.info("🙈 No face detected") belief_msg = BeliefMessage(delete=[{"name": "face_present", "arguments": []}])
# await self.post_belief("face_present", value=False)
msg = InternalMessage(
to=settings.agent_settings.bdi_core_name,
sender=self.name,
thread="beliefs",
body=belief_msg.model_dump_json(),
)
await self.send(msg)
async def _update_face_belief(self, present: bool):
"""
Add or remove the `face_present` belief in the BDI Core Agent.
"""
if present:
payload = BeliefMessage(create=[Belief(name="face_present").model_dump()])
else:
payload = BeliefMessage(delete=[Belief(name="face_present").model_dump()])
message = InternalMessage(
to=settings.agent_settings.bdi_core_name,
sender=self.name,
thread="beliefs",
body=payload.model_dump_json(),
)
await self.send(message)

View File

@@ -11,7 +11,6 @@ from control_backend.schemas.belief_message import Belief, BeliefMessage
from control_backend.schemas.program import ConditionalNorm, Program from control_backend.schemas.program import ConditionalNorm, Program
from control_backend.schemas.ri_message import ( from control_backend.schemas.ri_message import (
GestureCommand, GestureCommand,
PauseCommand,
RIEndpoint, RIEndpoint,
SpeechCommand, SpeechCommand,
) )
@@ -385,8 +384,10 @@ class UserInterruptAgent(BaseAgent):
if pause == "true": if pause == "true":
# Send pause to VAD and VED agent # Send pause to VAD and VED agent
vad_message = InternalMessage( vad_message = InternalMessage(
to=[settings.agent_settings.vad_name, to=[
settings.agent_settings.visual_emotion_recognition_name], settings.agent_settings.vad_name,
settings.agent_settings.visual_emotion_recognition_name,
],
sender=self.name, sender=self.name,
body="PAUSE", body="PAUSE",
) )
@@ -396,8 +397,10 @@ class UserInterruptAgent(BaseAgent):
else: else:
# Send resume to VAD and VED agents # Send resume to VAD and VED agents
vad_message = InternalMessage( vad_message = InternalMessage(
to=[settings.agent_settings.vad_name, to=[
settings.agent_settings.visual_emotion_recognition_name], settings.agent_settings.vad_name,
settings.agent_settings.visual_emotion_recognition_name,
],
sender=self.name, sender=self.name,
body="RESUME", body="RESUME",
) )

View File

@@ -28,8 +28,8 @@ class LogicalOperator(Enum):
OR = "OR" OR = "OR"
type Belief = KeywordBelief | SemanticBelief | InferredBelief | EmotionBelief type Belief = KeywordBelief | SemanticBelief | InferredBelief | EmotionBelief | FaceBelief
type BasicBelief = KeywordBelief | SemanticBelief | EmotionBelief type BasicBelief = KeywordBelief | SemanticBelief | EmotionBelief | FaceBelief
class KeywordBelief(ProgramElement): class KeywordBelief(ProgramElement):
@@ -69,6 +69,7 @@ class InferredBelief(ProgramElement):
left: Belief left: Belief
right: Belief right: Belief
class EmotionBelief(ProgramElement): class EmotionBelief(ProgramElement):
""" """
Represents a belief that is set when a certain emotion is detected. Represents a belief that is set when a certain emotion is detected.
@@ -79,6 +80,17 @@ class EmotionBelief(ProgramElement):
name: str = "" name: str = ""
emotion: str emotion: str
class FaceBelief(ProgramElement):
"""
Represents the belief that at least one face is currently detected.
This belief is maintained by a perception agent (not inferred).
"""
face_present: bool
name: str = ""
class Norm(ProgramElement): class Norm(ProgramElement):
""" """
Base class for behavioral norms that guide the robot's interactions. Base class for behavioral norms that guide the robot's interactions.