feat: face recognition agent #53
@@ -23,6 +23,7 @@ from control_backend.schemas.program import (
|
|||||||
BasicNorm,
|
BasicNorm,
|
||||||
ConditionalNorm,
|
ConditionalNorm,
|
||||||
EmotionBelief,
|
EmotionBelief,
|
||||||
|
FaceBelief,
|
||||||
GestureAction,
|
GestureAction,
|
||||||
Goal,
|
Goal,
|
||||||
InferredBelief,
|
InferredBelief,
|
||||||
@@ -465,6 +466,10 @@ class AgentSpeakGenerator:
|
|||||||
def _(self, eb: EmotionBelief) -> AstExpression:
|
def _(self, eb: EmotionBelief) -> AstExpression:
|
||||||
return AstLiteral("emotion_detected", [AstAtom(eb.emotion)])
|
return AstLiteral("emotion_detected", [AstAtom(eb.emotion)])
|
||||||
|
|
||||||
|
@_astify.register
|
||||||
|
def _(self, eb: FaceBelief) -> AstExpression:
|
||||||
|
return AstLiteral("face_present")
|
||||||
|
|
||||||
@_astify.register
|
@_astify.register
|
||||||
def _(self, ib: InferredBelief) -> AstExpression:
|
def _(self, ib: InferredBelief) -> AstExpression:
|
||||||
return AstBinaryOp(
|
return AstBinaryOp(
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
from random import random
|
||||||
|
|
||||||
import zmq
|
import zmq
|
||||||
import zmq.asyncio as azmq
|
import zmq.asyncio as azmq
|
||||||
from zmq.asyncio import Context
|
from zmq.asyncio import Context
|
||||||
|
|
||||||
from control_backend.agents import BaseAgent
|
from control_backend.agents import BaseAgent
|
||||||
|
from control_backend.core.agent_system import InternalMessage
|
||||||
|
from control_backend.core.config import settings
|
||||||
|
from control_backend.schemas.belief_message import Belief, BeliefMessage
|
||||||
|
|
||||||
|
|
||||||
class FacePerceptionAgent(BaseAgent):
|
class FacePerceptionAgent(BaseAgent):
|
||||||
@@ -51,19 +55,49 @@ class FacePerceptionAgent(BaseAgent):
|
|||||||
|
|
||||||
if face_present != self._last_face_state:
|
if face_present != self._last_face_state:
|
||||||
self._last_face_state = face_present
|
self._last_face_state = face_present
|
||||||
self.logger.info("👀 Face detected" if face_present else "🙈 Face lost")
|
self.logger.debug("Face detected" if face_present else "Face lost")
|
||||||
# TODO: post belief to BDI here
|
await self._update_face_belief(face_present)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.warning("Face polling failed")
|
self.logger.warning("Face polling failed")
|
||||||
self.logger.warn(e)
|
self.logger.warn(e)
|
||||||
|
i = random()
|
||||||
|
await self._update_face_belief(i > 0.5)
|
||||||
|
|
||||||
await asyncio.sleep(poll_interval)
|
await asyncio.sleep(poll_interval)
|
||||||
|
|
||||||
async def _handle_face_change(self, present: bool):
|
async def _post_face_belief(self, present: bool):
|
||||||
|
"""
|
||||||
|
Send a face_present belief update to the BDI Core Agent.
|
||||||
|
"""
|
||||||
if present:
|
if present:
|
||||||
self.logger.info("👀 Face detected")
|
belief_msg = BeliefMessage(create=[{"name": "face_present", "arguments": []}])
|
||||||
# await self.post_belief("face_present", value=True)
|
|
||||||
else:
|
else:
|
||||||
self.logger.info("🙈 No face detected")
|
belief_msg = BeliefMessage(delete=[{"name": "face_present", "arguments": []}])
|
||||||
# await self.post_belief("face_present", value=False)
|
|
||||||
|
msg = InternalMessage(
|
||||||
|
to=settings.agent_settings.bdi_core_name,
|
||||||
|
sender=self.name,
|
||||||
|
thread="beliefs",
|
||||||
|
body=belief_msg.model_dump_json(),
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.send(msg)
|
||||||
|
|
||||||
|
async def _update_face_belief(self, present: bool):
|
||||||
|
"""
|
||||||
|
Add or remove the `face_present` belief in the BDI Core Agent.
|
||||||
|
"""
|
||||||
|
if present:
|
||||||
|
payload = BeliefMessage(create=[Belief(name="face_present").model_dump()])
|
||||||
|
else:
|
||||||
|
payload = BeliefMessage(delete=[Belief(name="face_present").model_dump()])
|
||||||
|
|
||||||
|
message = InternalMessage(
|
||||||
|
to=settings.agent_settings.bdi_core_name,
|
||||||
|
sender=self.name,
|
||||||
|
thread="beliefs",
|
||||||
|
body=payload.model_dump_json(),
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.send(message)
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ from control_backend.schemas.belief_message import Belief, BeliefMessage
|
|||||||
from control_backend.schemas.program import ConditionalNorm, Program
|
from control_backend.schemas.program import ConditionalNorm, Program
|
||||||
from control_backend.schemas.ri_message import (
|
from control_backend.schemas.ri_message import (
|
||||||
GestureCommand,
|
GestureCommand,
|
||||||
PauseCommand,
|
|
||||||
RIEndpoint,
|
RIEndpoint,
|
||||||
SpeechCommand,
|
SpeechCommand,
|
||||||
)
|
)
|
||||||
@@ -385,8 +384,10 @@ class UserInterruptAgent(BaseAgent):
|
|||||||
if pause == "true":
|
if pause == "true":
|
||||||
# Send pause to VAD and VED agent
|
# Send pause to VAD and VED agent
|
||||||
vad_message = InternalMessage(
|
vad_message = InternalMessage(
|
||||||
to=[settings.agent_settings.vad_name,
|
to=[
|
||||||
settings.agent_settings.visual_emotion_recognition_name],
|
settings.agent_settings.vad_name,
|
||||||
|
settings.agent_settings.visual_emotion_recognition_name,
|
||||||
|
],
|
||||||
sender=self.name,
|
sender=self.name,
|
||||||
body="PAUSE",
|
body="PAUSE",
|
||||||
)
|
)
|
||||||
@@ -396,8 +397,10 @@ class UserInterruptAgent(BaseAgent):
|
|||||||
else:
|
else:
|
||||||
# Send resume to VAD and VED agents
|
# Send resume to VAD and VED agents
|
||||||
vad_message = InternalMessage(
|
vad_message = InternalMessage(
|
||||||
to=[settings.agent_settings.vad_name,
|
to=[
|
||||||
settings.agent_settings.visual_emotion_recognition_name],
|
settings.agent_settings.vad_name,
|
||||||
|
settings.agent_settings.visual_emotion_recognition_name,
|
||||||
|
],
|
||||||
sender=self.name,
|
sender=self.name,
|
||||||
body="RESUME",
|
body="RESUME",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -28,8 +28,8 @@ class LogicalOperator(Enum):
|
|||||||
OR = "OR"
|
OR = "OR"
|
||||||
|
|
||||||
|
|
||||||
type Belief = KeywordBelief | SemanticBelief | InferredBelief | EmotionBelief
|
type Belief = KeywordBelief | SemanticBelief | InferredBelief | EmotionBelief | FaceBelief
|
||||||
type BasicBelief = KeywordBelief | SemanticBelief | EmotionBelief
|
type BasicBelief = KeywordBelief | SemanticBelief | EmotionBelief | FaceBelief
|
||||||
|
|
||||||
|
|
||||||
class KeywordBelief(ProgramElement):
|
class KeywordBelief(ProgramElement):
|
||||||
@@ -69,6 +69,7 @@ class InferredBelief(ProgramElement):
|
|||||||
left: Belief
|
left: Belief
|
||||||
right: Belief
|
right: Belief
|
||||||
|
|
||||||
|
|
||||||
class EmotionBelief(ProgramElement):
|
class EmotionBelief(ProgramElement):
|
||||||
"""
|
"""
|
||||||
Represents a belief that is set when a certain emotion is detected.
|
Represents a belief that is set when a certain emotion is detected.
|
||||||
@@ -79,6 +80,17 @@ class EmotionBelief(ProgramElement):
|
|||||||
name: str = ""
|
name: str = ""
|
||||||
emotion: str
|
emotion: str
|
||||||
|
|
||||||
|
|
||||||
|
class FaceBelief(ProgramElement):
|
||||||
|
"""
|
||||||
|
Represents the belief that at least one face is currently detected.
|
||||||
|
This belief is maintained by a perception agent (not inferred).
|
||||||
|
"""
|
||||||
|
|
||||||
|
face_present: bool
|
||||||
|
name: str = ""
|
||||||
|
|
||||||
|
|
||||||
class Norm(ProgramElement):
|
class Norm(ProgramElement):
|
||||||
"""
|
"""
|
||||||
Base class for behavioral norms that guide the robot's interactions.
|
Base class for behavioral norms that guide the robot's interactions.
|
||||||
|
|||||||
Reference in New Issue
Block a user