From d043c543367082aa70db2196fc548d305f0e3a9f Mon Sep 17 00:00:00 2001 From: Kasper Date: Tue, 16 Dec 2025 10:21:50 +0100 Subject: [PATCH 01/90] refactor: program restructure Also includes some AgentSpeak generation. ref: N25B-376 --- pyproject.toml | 1 + .../agents/bdi/bdi_program_manager.py | 373 ++++++++++++++++-- src/control_backend/schemas/program.py | 204 ++++++++-- uv.lock | 23 ++ 4 files changed, 532 insertions(+), 69 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e57a03c..cdc2ce3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "pydantic>=2.12.0", "pydantic-settings>=2.11.0", "python-json-logger>=4.0.0", + "python-slugify>=8.0.4", "pyyaml>=6.0.3", "pyzmq>=27.1.0", "silero-vad>=6.0.0", diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 83dea93..4213cfa 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -1,12 +1,311 @@ import zmq from pydantic import ValidationError +from slugify import slugify from zmq.asyncio import Context from control_backend.agents import BaseAgent -from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings -from control_backend.schemas.belief_message import Belief, BeliefMessage -from control_backend.schemas.program import Program +from control_backend.schemas.program import ( + Action, + BasicBelief, + BasicNorm, + Belief, + ConditionalNorm, + GestureAction, + Goal, + InferredBelief, + KeywordBelief, + LLMAction, + LogicalOperator, + Phase, + Plan, + Program, + ProgramElement, + SemanticBelief, + SpeechAction, +) + +test_program = Program( + phases=[ + Phase( + norms=[ + BasicNorm(norm="Talk like a pirate"), + ConditionalNorm( + condition=InferredBelief( + left=KeywordBelief(keyword="Arr"), + right=SemanticBelief(description="testing", name="semantic belief"), + operator=LogicalOperator.OR, + name="Talking to a pirate", + ), + norm="Use nautical terms", + ), + ConditionalNorm( + condition=SemanticBelief( + description="We are talking to a child", name="talking to child" + ), + norm="Do not use cuss words", + ), + ], + triggers=[ + # Trigger( + # condition=InferredBelief( + # left=KeywordBelief(keyword="key"), + # right=InferredBelief( + # left=KeywordBelief(keyword="key2"), + # right=SemanticBelief( + # description="Decode this", name="semantic belief 2" + # ), + # operator=LogicalOperator.OR, + # name="test trigger inferred inner", + # ), + # operator=LogicalOperator.OR, + # name="test trigger inferred outer", + # ), + # plan=Plan(steps=[]), + # ) + ], + goals=[ + Goal( + name="Determine user age", + plan=Plan(steps=[LLMAction(goal="Determine the age of the user.")]), + ), + Goal( + name="Find the user's name", + plan=Plan( + steps=[ + Goal( + name="Greet the user", + plan=Plan(steps=[LLMAction(goal="Greet the user.")]), + can_fail=False, + ), + Goal( + name="Ask for name", + plan=Plan(steps=[LLMAction(goal="Obtain the user's name.")]), + ), + ] + ), + ), + Goal(name="Tell a joke", plan=Plan(steps=[LLMAction(goal="Tell a joke.")])), + ], + id=1, + ) + ] +) + + +class AgentSpeakGenerator: + """ + Converts Pydantic representation of behavior programs into AgentSpeak(L) code string. + """ + + def generate(self, program: Program) -> str: + lines = [] + lines.append("") + + lines += self._generate_initial_beliefs(program) + + lines += self._generate_norms(program) + + lines += self._generate_belief_inference(program) + + lines += self._generate_goals(program) + + lines += self._generate_triggers(program) + + return "\n".join(lines) + + def _generate_initial_beliefs(self, program: Program) -> list[str]: + lines = [] + lines.append("// --- Initial beliefs ---") + + lines.append(f"phase({program.phases[0].id}).") + + lines += ["", ""] + + return lines + + def _generate_norms(self, program: Program) -> list[str]: + lines = [] + lines.append("// --- Norms ---") + + for phase in program.phases: + for norm in phase.norms: + if type(norm) is BasicNorm: + lines.append(f"{self._slugify(norm)} :- phase({phase.id}).") + if type(norm) is ConditionalNorm: + lines.append( + f"{self._slugify(norm)} :- phase({phase.id}) & " + f"{self._slugify(norm.condition)}." + ) + + lines += ["", ""] + + return lines + + def _generate_belief_inference(self, program: Program) -> list[str]: + lines = [] + lines.append("// --- Belief inference rules ---") + + for phase in program.phases: + for norm in phase.norms: + if not isinstance(norm, ConditionalNorm): + continue + + lines += self._belief_inference_recursive(norm.condition) + + for trigger in phase.triggers: + lines += self._belief_inference_recursive(trigger.condition) + + lines += ["", ""] + + return lines + + def _belief_inference_recursive(self, belief: Belief) -> list[str]: + lines = [] + + if type(belief) is KeywordBelief: + lines.append( + f"{self._slugify(belief)} :- user_said(Message) & " + f'.substring(Message, "{belief.keyword}", Pos) & Pos >= 0.' + ) + if type(belief) is InferredBelief: + lines.append( + f"{self._slugify(belief)} :- {self._slugify(belief.left)} " + f"{'&' if belief.operator == LogicalOperator.AND else '|'} " + f"{self._slugify(belief.right)}." + ) + lines += self._belief_inference_recursive(belief.left) + lines += self._belief_inference_recursive(belief.right) + + return lines + + def _generate_goals(self, program: Program) -> list[str]: + lines = [] + lines.append("// --- Goals ---") + + for phase in program.phases: + previous_goal: Goal | None = None + for goal in phase.goals: + lines += self._generate_plan_recursive(goal, phase, previous_goal) + previous_goal = goal + + lines += ["", ""] + return lines + + def _generate_plan_recursive( + self, goal: Goal, phase: Phase, previous_goal: Goal | None = None + ) -> list[str]: + lines = [] + lines.append(f"+{self._slugify(goal, include_prefix=True)}") + + # Context + lines.append(f"{' ' * 2}:{' ' * 3}phase({phase.id}) &") + lines.append(f"{' ' * 6}not responded_this_turn &") + lines.append(f"{' ' * 6}not achieved_{self._slugify(goal)} &") + if previous_goal: + lines.append(f"{' ' * 6}achieved_{self._slugify(previous_goal)}") + else: + lines.append(f"{' ' * 6}true") + + extra_goals_to_generate = [] + + steps = goal.plan.steps + first_step = steps[0] + lines.append( + f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" + f"{'.' if len(steps) == 1 and goal.can_fail else ';'}" + ) + if isinstance(first_step, Goal): + extra_goals_to_generate.append(first_step) + + for step in steps[1:-1]: + lines.append(f"{' ' * 6}{self._slugify(step, include_prefix=True)};") + if isinstance(step, Goal): + extra_goals_to_generate.append(step) + + if len(steps) > 1: + last_step = steps[-1] + lines.append( + f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}" + f"{'.' if goal.can_fail else ';'}" + ) + if isinstance(last_step, Goal): + extra_goals_to_generate.append(last_step) + + if not goal.can_fail: + lines.append(f"{' ' * 6}+achieved_{self._slugify(goal)}.") + + lines.append("") + + extra_previous_goal: Goal | None = None + for extra_goal in extra_goals_to_generate: + lines += self._generate_plan_recursive(extra_goal, phase, extra_previous_goal) + extra_previous_goal = extra_goal + + return lines + + def _generate_triggers(self, program: Program) -> list[str]: + lines = [] + lines.append("// --- Triggers ---") + + lines += ["", ""] + return lines + + def _slugify(self, element: ProgramElement, include_prefix: bool = False) -> str: + def base_slugify_call(text: str): + return slugify(text, separator="_", stopwords=["a", "the"]) + + if type(element) is KeywordBelief: + return f'keyword_said("{element.keyword}")' + + if type(element) is SemanticBelief: + name = element.name + return f"semantic_{base_slugify_call(name if name else element.description)}" + + if isinstance(element, BasicNorm): + return f'norm("{element.norm}")' + + if isinstance(element, Goal): + return f"{'!' if include_prefix else ''}{base_slugify_call(element.name)}" + + if isinstance(element, SpeechAction): + return f'.say("{element.text}")' + + if isinstance(element, GestureAction): + return f'.gesture("{element.gesture}")' + + if isinstance(element, LLMAction): + return f'!generate_response_with_goal("{element.goal}")' + + if isinstance(element, Action.__value__): + raise NotImplementedError( + "Have not implemented an ASL string representation for this action." + ) + + if element.name == "": + raise ValueError("Name must be initialized for this type of ProgramElement.") + + return base_slugify_call(element.name) + + def _extract_basic_beliefs_from_program(self, program: Program) -> list[BasicBelief]: + beliefs = [] + + for phase in program.phases: + for norm in phase.norms: + if isinstance(norm, ConditionalNorm): + beliefs += self._extract_basic_beliefs_from_belief(norm.condition) + + for trigger in phase.triggers: + beliefs += self._extract_basic_beliefs_from_belief(trigger.condition) + + return beliefs + + def _extract_basic_beliefs_from_belief(self, belief: Belief) -> list[BasicBelief]: + if isinstance(belief, InferredBelief): + return self._extract_basic_beliefs_from_belief( + belief.left + ) + self._extract_basic_beliefs_from_belief(belief.right) + return [belief] class BDIProgramManager(BaseAgent): @@ -25,40 +324,40 @@ class BDIProgramManager(BaseAgent): super().__init__(**kwargs) self.sub_socket = None - async def _send_to_bdi(self, program: Program): - """ - Convert a received program into BDI beliefs and send them to the BDI Core Agent. - - Currently, it takes the **first phase** of the program and extracts: - - **Norms**: Constraints or rules the agent must follow. - - **Goals**: Objectives the agent must achieve. - - These are sent as a ``BeliefMessage`` with ``replace=True``, meaning they will - overwrite any existing norms/goals of the same name in the BDI agent. - - :param program: The program object received from the API. - """ - first_phase = program.phases[0] - norms_belief = Belief( - name="norms", - arguments=[norm.norm for norm in first_phase.norms], - replace=True, - ) - goals_belief = Belief( - name="goals", - arguments=[goal.description for goal in first_phase.goals], - replace=True, - ) - program_beliefs = BeliefMessage(beliefs=[norms_belief, goals_belief]) - - message = InternalMessage( - to=settings.agent_settings.bdi_core_name, - sender=self.name, - body=program_beliefs.model_dump_json(), - thread="beliefs", - ) - await self.send(message) - self.logger.debug("Sent new norms and goals to the BDI agent.") + # async def _send_to_bdi(self, program: Program): + # """ + # Convert a received program into BDI beliefs and send them to the BDI Core Agent. + # + # Currently, it takes the **first phase** of the program and extracts: + # - **Norms**: Constraints or rules the agent must follow. + # - **Goals**: Objectives the agent must achieve. + # + # These are sent as a ``BeliefMessage`` with ``replace=True``, meaning they will + # overwrite any existing norms/goals of the same name in the BDI agent. + # + # :param program: The program object received from the API. + # """ + # first_phase = program.phases[0] + # norms_belief = Belief( + # name="norms", + # arguments=[norm.norm for norm in first_phase.norms], + # replace=True, + # ) + # goals_belief = Belief( + # name="goals", + # arguments=[goal.description for goal in first_phase.goals], + # replace=True, + # ) + # program_beliefs = BeliefMessage(beliefs=[norms_belief, goals_belief]) + # + # message = InternalMessage( + # to=settings.agent_settings.bdi_core_name, + # sender=self.name, + # body=program_beliefs.model_dump_json(), + # thread="beliefs", + # ) + # await self.send(message) + # self.logger.debug("Sent new norms and goals to the BDI agent.") async def _receive_programs(self): """ diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 28969b9..d02923e 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -1,64 +1,204 @@ +from enum import Enum + from pydantic import BaseModel -class Norm(BaseModel): +class ProgramElement(BaseModel): """ - Represents a behavioral norm. + Represents a basic element of our behavior program. + :ivar name: The researcher-assigned name of the element. :ivar id: Unique identifier. - :ivar label: Human-readable label. - :ivar norm: The actual norm text describing the behavior. """ - id: str - label: str - norm: str + name: str + id: int -class Goal(BaseModel): +class LogicalOperator(Enum): + AND = "AND" + OR = "OR" + + +type Belief = KeywordBelief | SemanticBelief | InferredBelief +type BasicBelief = KeywordBelief | SemanticBelief + + +class KeywordBelief(ProgramElement): """ - Represents an objective to be achieved. + Represents a belief that is set when the user spoken text contains a certain keyword. - :ivar id: Unique identifier. - :ivar label: Human-readable label. - :ivar description: Detailed description of the goal. - :ivar achieved: Status flag indicating if the goal has been met. + :ivar keyword: The keyword on which this belief gets set. """ - id: str - label: str - description: str - achieved: bool - - -class TriggerKeyword(BaseModel): - id: str + name: str = "" + id: int = -1 keyword: str -class KeywordTrigger(BaseModel): - id: str - label: str - type: str - keywords: list[TriggerKeyword] +class SemanticBelief(ProgramElement): + """ + Represents a belief that is set by semantic LLM validation. + + :ivar description: Description of how to form the belief, used by the LLM. + """ + + name: str = "" + id: int = -1 + description: str -class Phase(BaseModel): +class InferredBelief(ProgramElement): + """ + Represents a belief that gets formed by combining two beliefs with a logical AND or OR. + + These beliefs can also be :class:`InferredBelief`, leading to arbitrarily deep nesting. + + :ivar operator: The logical operator to apply. + :ivar left: The left part of the logical expression. + :ivar right: The right part of the logical expression. + """ + + name: str = "" + id: int = -1 + operator: LogicalOperator + left: Belief + right: Belief + + +type Norm = BasicNorm | ConditionalNorm + + +class BasicNorm(ProgramElement): + """ + Represents a behavioral norm. + + :ivar norm: The actual norm text describing the behavior. + :ivar critical: When true, this norm should absolutely not be violated (checked separately). + """ + + name: str = "" + id: int = -1 + norm: str + critical: bool = False + + +class ConditionalNorm(BasicNorm): + """ + Represents a norm that is only active when a condition is met (i.e., a certain belief holds). + + :ivar condition: When to activate this norm. + """ + + name: str = "" + id: int = -1 + condition: Belief + + +type PlanElement = Goal | Action + + +class Plan(ProgramElement): + """ + Represents a list of steps to execute. Each of these steps can be a goal (with its own plan) + or a simple action. + + :ivar steps: The actions or subgoals to execute, in order. + """ + + name: str = "" + id: int = -1 + steps: list[PlanElement] + + +class Goal(ProgramElement): + """ + Represents an objective to be achieved. To reach the goal, we should execute + the corresponding plan. If we can fail to achieve a goal after executing the plan, + for example when the achieving of the goal is dependent on the user's reply, this means + that the achieved status will be set from somewhere else in the program. + + :ivar plan: The plan to execute. + :ivar can_fail: Whether we can fail to achieve the goal after executing the plan. + """ + + id: int = -1 + plan: Plan + can_fail: bool = True + + +type Action = SpeechAction | GestureAction | LLMAction + + +class SpeechAction(ProgramElement): + """ + Represents the action of the robot speaking a literal text. + + :ivar text: The text to speak. + """ + + name: str = "" + id: int = -1 + text: str + + +# TODO: gestures +class Gesture(Enum): + RAISE_HAND = "RAISE_HAND" + + +class GestureAction(ProgramElement): + """ + Represents the action of the robot performing a gesture. + + :ivar gesture: The gesture to perform. + """ + + name: str = "" + id: int = -1 + gesture: Gesture + + +class LLMAction(ProgramElement): + """ + Represents the action of letting an LLM generate a reply based on its chat history + and an additional goal added in the prompt. + + :ivar goal: The extra (temporary) goal to add to the LLM. + """ + + name: str = "" + id: int = -1 + goal: str + + +class Trigger(ProgramElement): + """ + Represents a belief-based trigger. When a belief is set, the corresponding plan is executed. + + :ivar condition: When to activate the trigger. + :ivar plan: The plan to execute. + """ + + name: str = "" + id: int = -1 + condition: Belief + plan: Plan + + +class Phase(ProgramElement): """ A distinct phase within a program, containing norms, goals, and triggers. - :ivar id: Unique identifier. - :ivar label: Human-readable label. :ivar norms: List of norms active in this phase. :ivar goals: List of goals to pursue in this phase. :ivar triggers: List of triggers that define transitions out of this phase. """ - id: str - label: str + name: str = "" norms: list[Norm] goals: list[Goal] - triggers: list[KeywordTrigger] + triggers: list[Trigger] class Program(BaseModel): diff --git a/uv.lock b/uv.lock index ff4b8a7..ce46ceb 100644 --- a/uv.lock +++ b/uv.lock @@ -997,6 +997,7 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-json-logger" }, + { name = "python-slugify" }, { name = "pyyaml" }, { name = "pyzmq" }, { name = "silero-vad" }, @@ -1046,6 +1047,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.12.0" }, { name = "pydantic-settings", specifier = ">=2.11.0" }, { name = "python-json-logger", specifier = ">=4.0.0" }, + { name = "python-slugify", specifier = ">=8.0.4" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "pyzmq", specifier = ">=27.1.0" }, { name = "silero-vad", specifier = ">=6.0.0" }, @@ -1341,6 +1343,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" }, ] +[[package]] +name = "python-slugify" +version = "8.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "text-unidecode" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/c7/5e1547c44e31da50a460df93af11a535ace568ef89d7a811069ead340c4a/python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856", size = 10921, upload-time = "2024-02-08T18:32:45.488Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", size = 10051, upload-time = "2024-02-08T18:32:43.911Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1864,6 +1878,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] +[[package]] +name = "text-unidecode" +version = "1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93", size = 76885, upload-time = "2019-08-30T21:36:45.405Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, +] + [[package]] name = "tiktoken" version = "0.12.0" From bab48006981f7a99614690da1fdf1d8e2ffacd4c Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Tue, 16 Dec 2025 12:10:52 +0100 Subject: [PATCH 02/90] feat: add trigger generation ref: N25B-376 --- .../agents/bdi/bdi_program_manager.py | 104 ++++++++++++++---- 1 file changed, 85 insertions(+), 19 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 4213cfa..5b2d484 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -23,6 +23,7 @@ from control_backend.schemas.program import ( ProgramElement, SemanticBelief, SpeechAction, + Trigger, ) test_program = Program( @@ -47,22 +48,30 @@ test_program = Program( ), ], triggers=[ - # Trigger( - # condition=InferredBelief( - # left=KeywordBelief(keyword="key"), - # right=InferredBelief( - # left=KeywordBelief(keyword="key2"), - # right=SemanticBelief( - # description="Decode this", name="semantic belief 2" - # ), - # operator=LogicalOperator.OR, - # name="test trigger inferred inner", - # ), - # operator=LogicalOperator.OR, - # name="test trigger inferred outer", - # ), - # plan=Plan(steps=[]), - # ) + Trigger( + condition=InferredBelief( + left=KeywordBelief(keyword="key"), + right=InferredBelief( + left=KeywordBelief(keyword="key2"), + right=SemanticBelief( + description="Decode this", name="semantic belief 2" + ), + operator=LogicalOperator.OR, + name="test trigger inferred inner", + ), + operator=LogicalOperator.OR, + name="test trigger inferred outer", + ), + plan=Plan( + steps=[ + SpeechAction(text="Testing trigger"), + Goal( + name="Testing trigger", + plan=Plan(steps=[LLMAction(goal="Do something")]), + ), + ] + ), + ) ], goals=[ Goal( @@ -93,6 +102,10 @@ test_program = Program( ) +def do_things(): + print(AgentSpeakGenerator().generate(test_program)) + + class AgentSpeakGenerator: """ Converts Pydantic representation of behavior programs into AgentSpeak(L) code string. @@ -186,13 +199,13 @@ class AgentSpeakGenerator: for phase in program.phases: previous_goal: Goal | None = None for goal in phase.goals: - lines += self._generate_plan_recursive(goal, phase, previous_goal) + lines += self._generate_goal_plan_recursive(goal, phase, previous_goal) previous_goal = goal lines += ["", ""] return lines - def _generate_plan_recursive( + def _generate_goal_plan_recursive( self, goal: Goal, phase: Phase, previous_goal: Goal | None = None ) -> list[str]: lines = [] @@ -210,6 +223,11 @@ class AgentSpeakGenerator: extra_goals_to_generate = [] steps = goal.plan.steps + + if len(steps) == 0: + lines.append(f"{' ' * 2}<-{' ' * 2}true.") + return lines + first_step = steps[0] lines.append( f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" @@ -239,7 +257,7 @@ class AgentSpeakGenerator: extra_previous_goal: Goal | None = None for extra_goal in extra_goals_to_generate: - lines += self._generate_plan_recursive(extra_goal, phase, extra_previous_goal) + lines += self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) extra_previous_goal = extra_goal return lines @@ -248,9 +266,57 @@ class AgentSpeakGenerator: lines = [] lines.append("// --- Triggers ---") + for phase in program.phases: + for trigger in phase.triggers: + lines += self._generate_trigger_plan(trigger, phase) + lines += ["", ""] return lines + def _generate_trigger_plan(self, trigger: Trigger, phase: Phase) -> list[str]: + lines = [] + + belief_name = self._slugify(trigger.condition) + + lines.append(f"+{belief_name}") + lines.append(f"{' ' * 2}:{' ' * 3}phase({phase.id})") + + extra_goals_to_generate = [] + + steps = trigger.plan.steps + + if len(steps) == 0: + lines.append(f"{' ' * 2}<-{' ' * 2}true.") + return lines + + first_step = steps[0] + lines.append( + f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" + f"{'.' if len(steps) == 1 else ';'}" + ) + if isinstance(first_step, Goal): + extra_goals_to_generate.append(first_step) + + for step in steps[1:-1]: + lines.append(f"{' ' * 6}{self._slugify(step, include_prefix=True)};") + if isinstance(step, Goal): + extra_goals_to_generate.append(step) + + if len(steps) > 1: + last_step = steps[-1] + lines.append(f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}.") + if isinstance(last_step, Goal): + extra_goals_to_generate.append(last_step) + + lines.append("") + + extra_previous_goal: Goal | None = None + for extra_goal in extra_goals_to_generate: + lines += self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) + extra_previous_goal = extra_goal + + return lines + def _slugify(self, element: ProgramElement, include_prefix: bool = False) -> str: def base_slugify_call(text: str): return slugify(text, separator="_", stopwords=["a", "the"]) From 4a432a603fbdd2c19690dc253e44ab2e2ffeb715 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Tue, 16 Dec 2025 14:12:04 +0100 Subject: [PATCH 03/90] fix: separate trigger plan generation ref: N25B-376 --- .../agents/bdi/bdi_program_manager.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 5b2d484..8f5bf03 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -310,6 +310,54 @@ class AgentSpeakGenerator: lines.append("") + extra_previous_goal: Goal | None = None + for extra_goal in extra_goals_to_generate: + lines += self._generate_trigger_plan_recursive(extra_goal, phase, extra_previous_goal) + extra_previous_goal = extra_goal + + return lines + + def _generate_trigger_plan_recursive( + self, goal: Goal, phase: Phase, previous_goal: Goal | None = None + ) -> list[str]: + lines = [] + lines.append(f"+{self._slugify(goal, include_prefix=True)}") + + extra_goals_to_generate = [] + + steps = goal.plan.steps + + if len(steps) == 0: + lines.append(f"{' ' * 2}<-{' ' * 2}true.") + return lines + + first_step = steps[0] + lines.append( + f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" + f"{'.' if len(steps) == 1 and goal.can_fail else ';'}" + ) + if isinstance(first_step, Goal): + extra_goals_to_generate.append(first_step) + + for step in steps[1:-1]: + lines.append(f"{' ' * 6}{self._slugify(step, include_prefix=True)};") + if isinstance(step, Goal): + extra_goals_to_generate.append(step) + + if len(steps) > 1: + last_step = steps[-1] + lines.append( + f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}" + f"{'.' if goal.can_fail else ';'}" + ) + if isinstance(last_step, Goal): + extra_goals_to_generate.append(last_step) + + if not goal.can_fail: + lines.append(f"{' ' * 6}+achieved_{self._slugify(goal)}.") + + lines.append("") + extra_previous_goal: Goal | None = None for extra_goal in extra_goals_to_generate: lines += self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) @@ -457,3 +505,7 @@ class BDIProgramManager(BaseAgent): self.sub_socket.subscribe("program") self.add_behavior(self._receive_programs()) + + +if __name__ == "__main__": + do_things() From 8cc177041ac5716e48888a3c0a1d8ce38df690c1 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Tue, 16 Dec 2025 15:12:22 +0100 Subject: [PATCH 04/90] feat: add a second phase in test_program ref: N25B-376 --- .../agents/bdi/bdi_program_manager.py | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 8f5bf03..2353fcb 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -97,7 +97,67 @@ test_program = Program( Goal(name="Tell a joke", plan=Plan(steps=[LLMAction(goal="Tell a joke.")])), ], id=1, - ) + ), + Phase( + id=2, + norms=[ + BasicNorm(norm="Use very gentle speech."), + ConditionalNorm( + condition=SemanticBelief( + description="We are talking to a child", name="talking to child" + ), + norm="Do not use cuss words", + ), + ], + triggers=[ + Trigger( + condition=InferredBelief( + left=KeywordBelief(keyword="help"), + right=SemanticBelief(description="User is stuck", name="stuck"), + operator=LogicalOperator.OR, + name="help_or_stuck", + ), + plan=Plan( + steps=[ + Goal( + name="Unblock user", + plan=Plan( + steps=[ + LLMAction( + goal="Provide a step-by-step path to " + "resolve the user's issue." + ) + ] + ), + ), + ] + ), + ), + ], + goals=[ + Goal( + name="Clarify intent", + plan=Plan( + steps=[ + LLMAction( + goal="Ask 1-2 targeted questions to clarify the " + "user's intent, then proceed." + ) + ] + ), + ), + Goal( + name="Provide solution", + plan=Plan( + steps=[LLMAction(goal="Deliver a solution to complete the user's goal.")] + ), + ), + Goal( + name="Summarize next steps", + plan=Plan(steps=[LLMAction(goal="Summarize what the user should do next.")]), + ), + ], + ), ] ) From 27f04f09588c21813aeb55100c400a59f5fb4267 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Tue, 16 Dec 2025 16:11:01 +0100 Subject: [PATCH 05/90] style: use yield instead of returning arrays ref: N25B-376 --- .../agents/bdi/bdi_program_manager.py | 152 ++++++++---------- 1 file changed, 64 insertions(+), 88 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 2353fcb..0eae52a 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -1,3 +1,5 @@ +from collections.abc import Iterable + import zmq from pydantic import ValidationError from slugify import slugify @@ -187,109 +189,94 @@ class AgentSpeakGenerator: return "\n".join(lines) - def _generate_initial_beliefs(self, program: Program) -> list[str]: - lines = [] - lines.append("// --- Initial beliefs ---") + def _generate_initial_beliefs(self, program: Program) -> Iterable[str]: + yield "// --- Initial beliefs ---" - lines.append(f"phase({program.phases[0].id}).") + yield f"phase({program.phases[0].id})." - lines += ["", ""] + yield from ["", ""] - return lines - - def _generate_norms(self, program: Program) -> list[str]: - lines = [] - lines.append("// --- Norms ---") + def _generate_norms(self, program: Program) -> Iterable[str]: + yield "// --- Norms ---" for phase in program.phases: for norm in phase.norms: if type(norm) is BasicNorm: - lines.append(f"{self._slugify(norm)} :- phase({phase.id}).") + yield f"{self._slugify(norm)} :- phase({phase.id})." if type(norm) is ConditionalNorm: - lines.append( + yield ( f"{self._slugify(norm)} :- phase({phase.id}) & " f"{self._slugify(norm.condition)}." ) - lines += ["", ""] + yield from ["", ""] - return lines - - def _generate_belief_inference(self, program: Program) -> list[str]: - lines = [] - lines.append("// --- Belief inference rules ---") + def _generate_belief_inference(self, program: Program) -> Iterable[str]: + yield "// --- Belief inference rules ---" for phase in program.phases: for norm in phase.norms: if not isinstance(norm, ConditionalNorm): continue - lines += self._belief_inference_recursive(norm.condition) + yield from self._belief_inference_recursive(norm.condition) for trigger in phase.triggers: - lines += self._belief_inference_recursive(trigger.condition) + yield from self._belief_inference_recursive(trigger.condition) - lines += ["", ""] - - return lines - - def _belief_inference_recursive(self, belief: Belief) -> list[str]: - lines = [] + yield from ["", ""] + def _belief_inference_recursive(self, belief: Belief) -> Iterable[str]: if type(belief) is KeywordBelief: - lines.append( + yield ( f"{self._slugify(belief)} :- user_said(Message) & " f'.substring(Message, "{belief.keyword}", Pos) & Pos >= 0.' ) if type(belief) is InferredBelief: - lines.append( + yield ( f"{self._slugify(belief)} :- {self._slugify(belief.left)} " f"{'&' if belief.operator == LogicalOperator.AND else '|'} " f"{self._slugify(belief.right)}." ) - lines += self._belief_inference_recursive(belief.left) - lines += self._belief_inference_recursive(belief.right) - return lines + yield from self._belief_inference_recursive(belief.left) + yield from self._belief_inference_recursive(belief.right) - def _generate_goals(self, program: Program) -> list[str]: - lines = [] - lines.append("// --- Goals ---") + def _generate_goals(self, program: Program) -> Iterable[str]: + yield "// --- Goals ---" for phase in program.phases: previous_goal: Goal | None = None for goal in phase.goals: - lines += self._generate_goal_plan_recursive(goal, phase, previous_goal) + yield from self._generate_goal_plan_recursive(goal, phase, previous_goal) previous_goal = goal - lines += ["", ""] - return lines + yield from ["", ""] def _generate_goal_plan_recursive( self, goal: Goal, phase: Phase, previous_goal: Goal | None = None - ) -> list[str]: - lines = [] - lines.append(f"+{self._slugify(goal, include_prefix=True)}") + ) -> Iterable[str]: + yield f"+{self._slugify(goal, include_prefix=True)}" # Context - lines.append(f"{' ' * 2}:{' ' * 3}phase({phase.id}) &") - lines.append(f"{' ' * 6}not responded_this_turn &") - lines.append(f"{' ' * 6}not achieved_{self._slugify(goal)} &") + yield f"{' ' * 2}:{' ' * 3}phase({phase.id}) &" + yield f"{' ' * 6}not responded_this_turn &" + yield f"{' ' * 6}not achieved_{self._slugify(goal)} &" if previous_goal: - lines.append(f"{' ' * 6}achieved_{self._slugify(previous_goal)}") + yield f"{' ' * 6}achieved_{self._slugify(previous_goal)}" else: - lines.append(f"{' ' * 6}true") + yield f"{' ' * 6}true" extra_goals_to_generate = [] steps = goal.plan.steps if len(steps) == 0: - lines.append(f"{' ' * 2}<-{' ' * 2}true.") - return lines + yield f"{' ' * 2}<-{' ' * 2}true." + return first_step = steps[0] - lines.append( + yield ( f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" f"{'.' if len(steps) == 1 and goal.can_fail else ';'}" ) @@ -297,13 +284,13 @@ class AgentSpeakGenerator: extra_goals_to_generate.append(first_step) for step in steps[1:-1]: - lines.append(f"{' ' * 6}{self._slugify(step, include_prefix=True)};") + yield f"{' ' * 6}{self._slugify(step, include_prefix=True)};" if isinstance(step, Goal): extra_goals_to_generate.append(step) if len(steps) > 1: last_step = steps[-1] - lines.append( + yield ( f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}" f"{'.' if goal.can_fail else ';'}" ) @@ -311,46 +298,40 @@ class AgentSpeakGenerator: extra_goals_to_generate.append(last_step) if not goal.can_fail: - lines.append(f"{' ' * 6}+achieved_{self._slugify(goal)}.") + yield f"{' ' * 6}+achieved_{self._slugify(goal)}." - lines.append("") + yield "" extra_previous_goal: Goal | None = None for extra_goal in extra_goals_to_generate: - lines += self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) + yield from self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) extra_previous_goal = extra_goal - return lines - - def _generate_triggers(self, program: Program) -> list[str]: - lines = [] - lines.append("// --- Triggers ---") + def _generate_triggers(self, program: Program) -> Iterable[str]: + yield "// --- Triggers ---" for phase in program.phases: for trigger in phase.triggers: - lines += self._generate_trigger_plan(trigger, phase) + yield from self._generate_trigger_plan(trigger, phase) - lines += ["", ""] - return lines - - def _generate_trigger_plan(self, trigger: Trigger, phase: Phase) -> list[str]: - lines = [] + yield from ["", ""] + def _generate_trigger_plan(self, trigger: Trigger, phase: Phase) -> Iterable[str]: belief_name = self._slugify(trigger.condition) - lines.append(f"+{belief_name}") - lines.append(f"{' ' * 2}:{' ' * 3}phase({phase.id})") + yield f"+{belief_name}" + yield f"{' ' * 2}:{' ' * 3}phase({phase.id})" extra_goals_to_generate = [] steps = trigger.plan.steps if len(steps) == 0: - lines.append(f"{' ' * 2}<-{' ' * 2}true.") - return lines + yield f"{' ' * 2}<-{' ' * 2}true." + return first_step = steps[0] - lines.append( + yield ( f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" f"{'.' if len(steps) == 1 else ';'}" ) @@ -358,41 +339,38 @@ class AgentSpeakGenerator: extra_goals_to_generate.append(first_step) for step in steps[1:-1]: - lines.append(f"{' ' * 6}{self._slugify(step, include_prefix=True)};") + yield f"{' ' * 6}{self._slugify(step, include_prefix=True)};" if isinstance(step, Goal): extra_goals_to_generate.append(step) if len(steps) > 1: last_step = steps[-1] - lines.append(f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}.") + yield f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}." if isinstance(last_step, Goal): extra_goals_to_generate.append(last_step) - lines.append("") + yield "" extra_previous_goal: Goal | None = None for extra_goal in extra_goals_to_generate: - lines += self._generate_trigger_plan_recursive(extra_goal, phase, extra_previous_goal) + yield from self._generate_trigger_plan_recursive(extra_goal, phase, extra_previous_goal) extra_previous_goal = extra_goal - return lines - def _generate_trigger_plan_recursive( self, goal: Goal, phase: Phase, previous_goal: Goal | None = None - ) -> list[str]: - lines = [] - lines.append(f"+{self._slugify(goal, include_prefix=True)}") + ) -> Iterable[str]: + yield f"+{self._slugify(goal, include_prefix=True)}" extra_goals_to_generate = [] steps = goal.plan.steps if len(steps) == 0: - lines.append(f"{' ' * 2}<-{' ' * 2}true.") - return lines + yield f"{' ' * 2}<-{' ' * 2}true." + return first_step = steps[0] - lines.append( + yield ( f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" f"{'.' if len(steps) == 1 and goal.can_fail else ';'}" ) @@ -400,13 +378,13 @@ class AgentSpeakGenerator: extra_goals_to_generate.append(first_step) for step in steps[1:-1]: - lines.append(f"{' ' * 6}{self._slugify(step, include_prefix=True)};") + yield f"{' ' * 6}{self._slugify(step, include_prefix=True)};" if isinstance(step, Goal): extra_goals_to_generate.append(step) if len(steps) > 1: last_step = steps[-1] - lines.append( + yield ( f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}" f"{'.' if goal.can_fail else ';'}" ) @@ -414,17 +392,15 @@ class AgentSpeakGenerator: extra_goals_to_generate.append(last_step) if not goal.can_fail: - lines.append(f"{' ' * 6}+achieved_{self._slugify(goal)}.") + yield f"{' ' * 6}+achieved_{self._slugify(goal)}." - lines.append("") + yield "" extra_previous_goal: Goal | None = None for extra_goal in extra_goals_to_generate: - lines += self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) + yield from self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) extra_previous_goal = extra_goal - return lines - def _slugify(self, element: ProgramElement, include_prefix: bool = False) -> str: def base_slugify_call(text: str): return slugify(text, separator="_", stopwords=["a", "the"]) From e704ec5ed44b3d645ea86f47715d790c67bda3a1 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Tue, 16 Dec 2025 17:00:32 +0100 Subject: [PATCH 06/90] feat: basic flow and phase transitions ref: N25B-376 --- .../agents/bdi/bdi_program_manager.py | 106 ++++++++++++++---- src/control_backend/schemas/program.py | 2 - 2 files changed, 85 insertions(+), 23 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 0eae52a..11c0b00 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -173,12 +173,20 @@ class AgentSpeakGenerator: Converts Pydantic representation of behavior programs into AgentSpeak(L) code string. """ + arrow_prefix = f"{' ' * 2}<-{' ' * 2}" + colon_prefix = f"{' ' * 2}:{' ' * 3}" + indent_prefix = " " * 6 + def generate(self, program: Program) -> str: lines = [] lines.append("") lines += self._generate_initial_beliefs(program) + lines += self._generate_basic_flow(program) + + lines += self._generate_phase_transitions(program) + lines += self._generate_norms(program) lines += self._generate_belief_inference(program) @@ -192,10 +200,60 @@ class AgentSpeakGenerator: def _generate_initial_beliefs(self, program: Program) -> Iterable[str]: yield "// --- Initial beliefs ---" - yield f"phase({program.phases[0].id})." + yield "phase(start)." yield from ["", ""] + def _generate_basic_flow(self, program: Program) -> Iterable[str]: + yield "// --- Basic flow ---" + + for phase in program.phases: + yield from self._generate_basic_flow_per_phase(phase) + + yield from ["", ""] + + def _generate_basic_flow_per_phase(self, phase: Phase) -> Iterable[str]: + yield "+user_said(Message)" + yield f"{self.colon_prefix}phase({phase.id})" + + goals = phase.goals + if goals: + yield f"{self.arrow_prefix}{self._slugify(goals[0], include_prefix=True)}" + for goal in goals[1:]: + yield f"{self.indent_prefix}{self._slugify(goal, include_prefix=True)}" + + yield f"{self.indent_prefix if goals else self.arrow_prefix}!transition_phase." + + def _generate_phase_transitions(self, program: Program) -> Iterable[str]: + yield "// --- Phase transitions ---" + + if len(program.phases) == 0: + yield from ["", ""] + return + + # TODO: remove outdated things + + for i in range(-1, len(program.phases)): + predecessor = program.phases[i] if i >= 0 else None + successor = program.phases[i + 1] if i < len(program.phases) - 1 else None + yield from self._generate_phase_transition(predecessor, successor) + + yield from self._generate_phase_transition(None, None) # to avoid failing plan + + yield from ["", ""] + + def _generate_phase_transition( + self, phase: Phase | None = None, next_phase: Phase | None = None + ) -> Iterable[str]: + yield "+!transition_phase" + + if phase is None and next_phase is None: # base case true to avoid failing plan + yield f"{self.arrow_prefix}true." + return + + yield f"{self.colon_prefix}phase({phase.id if phase else 'start'})" + yield f"{self.arrow_prefix}-+phase({next_phase.id if next_phase else 'end'})." + def _generate_norms(self, program: Program) -> Iterable[str]: yield "// --- Norms ---" @@ -259,46 +317,49 @@ class AgentSpeakGenerator: yield f"+{self._slugify(goal, include_prefix=True)}" # Context - yield f"{' ' * 2}:{' ' * 3}phase({phase.id}) &" - yield f"{' ' * 6}not responded_this_turn &" - yield f"{' ' * 6}not achieved_{self._slugify(goal)} &" + yield f"{self.colon_prefix}phase({phase.id}) &" + yield f"{self.indent_prefix}not responded_this_turn &" + yield f"{self.indent_prefix}not achieved_{self._slugify(goal)} &" if previous_goal: - yield f"{' ' * 6}achieved_{self._slugify(previous_goal)}" + yield f"{self.indent_prefix}achieved_{self._slugify(previous_goal)}" else: - yield f"{' ' * 6}true" + yield f"{self.indent_prefix}true" extra_goals_to_generate = [] steps = goal.plan.steps if len(steps) == 0: - yield f"{' ' * 2}<-{' ' * 2}true." + yield f"{self.arrow_prefix}true." return first_step = steps[0] yield ( - f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" + f"{self.arrow_prefix}{self._slugify(first_step, include_prefix=True)}" f"{'.' if len(steps) == 1 and goal.can_fail else ';'}" ) if isinstance(first_step, Goal): extra_goals_to_generate.append(first_step) for step in steps[1:-1]: - yield f"{' ' * 6}{self._slugify(step, include_prefix=True)};" + yield f"{self.indent_prefix}{self._slugify(step, include_prefix=True)};" if isinstance(step, Goal): extra_goals_to_generate.append(step) if len(steps) > 1: last_step = steps[-1] yield ( - f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}" + f"{self.indent_prefix}{self._slugify(last_step, include_prefix=True)}" f"{'.' if goal.can_fail else ';'}" ) if isinstance(last_step, Goal): extra_goals_to_generate.append(last_step) if not goal.can_fail: - yield f"{' ' * 6}+achieved_{self._slugify(goal)}." + yield f"{self.indent_prefix}+achieved_{self._slugify(goal)}." + + yield f"+{self._slugify(goal, include_prefix=True)}" + yield f"{self.arrow_prefix}true." yield "" @@ -320,32 +381,32 @@ class AgentSpeakGenerator: belief_name = self._slugify(trigger.condition) yield f"+{belief_name}" - yield f"{' ' * 2}:{' ' * 3}phase({phase.id})" + yield f"{self.colon_prefix}phase({phase.id})" extra_goals_to_generate = [] steps = trigger.plan.steps if len(steps) == 0: - yield f"{' ' * 2}<-{' ' * 2}true." + yield f"{self.arrow_prefix}true." return first_step = steps[0] yield ( - f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" + f"{self.arrow_prefix}{self._slugify(first_step, include_prefix=True)}" f"{'.' if len(steps) == 1 else ';'}" ) if isinstance(first_step, Goal): extra_goals_to_generate.append(first_step) for step in steps[1:-1]: - yield f"{' ' * 6}{self._slugify(step, include_prefix=True)};" + yield f"{self.indent_prefix}{self._slugify(step, include_prefix=True)};" if isinstance(step, Goal): extra_goals_to_generate.append(step) if len(steps) > 1: last_step = steps[-1] - yield f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}." + yield f"{self.indent_prefix}{self._slugify(last_step, include_prefix=True)}." if isinstance(last_step, Goal): extra_goals_to_generate.append(last_step) @@ -366,33 +427,36 @@ class AgentSpeakGenerator: steps = goal.plan.steps if len(steps) == 0: - yield f"{' ' * 2}<-{' ' * 2}true." + yield f"{self.arrow_prefix}true." return first_step = steps[0] yield ( - f"{' ' * 2}<-{' ' * 2}{self._slugify(first_step, include_prefix=True)}" + f"{self.arrow_prefix}{self._slugify(first_step, include_prefix=True)}" f"{'.' if len(steps) == 1 and goal.can_fail else ';'}" ) if isinstance(first_step, Goal): extra_goals_to_generate.append(first_step) for step in steps[1:-1]: - yield f"{' ' * 6}{self._slugify(step, include_prefix=True)};" + yield f"{self.indent_prefix}{self._slugify(step, include_prefix=True)};" if isinstance(step, Goal): extra_goals_to_generate.append(step) if len(steps) > 1: last_step = steps[-1] yield ( - f"{' ' * 6}{self._slugify(last_step, include_prefix=True)}" + f"{self.indent_prefix}{self._slugify(last_step, include_prefix=True)}" f"{'.' if goal.can_fail else ';'}" ) if isinstance(last_step, Goal): extra_goals_to_generate.append(last_step) if not goal.can_fail: - yield f"{' ' * 6}+achieved_{self._slugify(goal)}." + yield f"{self.indent_prefix}+achieved_{self._slugify(goal)}." + + yield f"+{self._slugify(goal, include_prefix=True)}" + yield f"{self.arrow_prefix}true." yield "" diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index d02923e..605694b 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -90,8 +90,6 @@ class ConditionalNorm(BasicNorm): :ivar condition: When to activate this norm. """ - name: str = "" - id: int = -1 condition: Belief From 742e36b94f3d10eebb456145e1b235d17c6f771f Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 17 Dec 2025 14:30:14 +0100 Subject: [PATCH 07/90] chore: non-optional uuid id ref: N25B-376 --- src/control_backend/schemas/program.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 605694b..7c73a6a 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -1,6 +1,6 @@ from enum import Enum -from pydantic import BaseModel +from pydantic import UUID4, BaseModel class ProgramElement(BaseModel): @@ -12,7 +12,7 @@ class ProgramElement(BaseModel): """ name: str - id: int + id: UUID4 class LogicalOperator(Enum): @@ -32,7 +32,6 @@ class KeywordBelief(ProgramElement): """ name: str = "" - id: int = -1 keyword: str @@ -44,7 +43,6 @@ class SemanticBelief(ProgramElement): """ name: str = "" - id: int = -1 description: str @@ -60,7 +58,6 @@ class InferredBelief(ProgramElement): """ name: str = "" - id: int = -1 operator: LogicalOperator left: Belief right: Belief @@ -78,7 +75,6 @@ class BasicNorm(ProgramElement): """ name: str = "" - id: int = -1 norm: str critical: bool = False @@ -105,7 +101,6 @@ class Plan(ProgramElement): """ name: str = "" - id: int = -1 steps: list[PlanElement] @@ -120,7 +115,6 @@ class Goal(ProgramElement): :ivar can_fail: Whether we can fail to achieve the goal after executing the plan. """ - id: int = -1 plan: Plan can_fail: bool = True @@ -136,7 +130,6 @@ class SpeechAction(ProgramElement): """ name: str = "" - id: int = -1 text: str @@ -153,7 +146,6 @@ class GestureAction(ProgramElement): """ name: str = "" - id: int = -1 gesture: Gesture @@ -166,7 +158,6 @@ class LLMAction(ProgramElement): """ name: str = "" - id: int = -1 goal: str @@ -179,7 +170,6 @@ class Trigger(ProgramElement): """ name: str = "" - id: int = -1 condition: Belief plan: Plan From 1d36d2e08951e40ded764860d13214e8135f294c Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 17 Dec 2025 15:33:27 +0100 Subject: [PATCH 08/90] feat: (hopefully) better intermediate representation ref: N25B-376 --- src/control_backend/agents/bdi/asl_ast.py | 172 ++++++++++ src/control_backend/agents/bdi/asl_gen.py | 295 ++++++++++++++++++ .../agents/bdi/bdi_program_manager.py | 131 ++++++-- src/control_backend/schemas/program.py | 15 +- 4 files changed, 581 insertions(+), 32 deletions(-) create mode 100644 src/control_backend/agents/bdi/asl_ast.py create mode 100644 src/control_backend/agents/bdi/asl_gen.py diff --git a/src/control_backend/agents/bdi/asl_ast.py b/src/control_backend/agents/bdi/asl_ast.py new file mode 100644 index 0000000..6543b63 --- /dev/null +++ b/src/control_backend/agents/bdi/asl_ast.py @@ -0,0 +1,172 @@ +import typing +from dataclasses import dataclass, field + +# --- Types --- + + +@dataclass +class BeliefLiteral: + """ + Represents a literal or atom. + Example: phase(1), user_said("hello"), ~started + """ + + functor: str + args: list[str] = field(default_factory=list) + negated: bool = False + + def __str__(self): + # In ASL, 'not' is usually for closed-world assumption (prolog style), + # '~' is for explicit negation in beliefs. + # For simplicity in behavior trees, we often use 'not' for conditions. + prefix = "not " if self.negated else "" + if not self.args: + return f"{prefix}{self.functor}" + + # Clean args to ensure strings are quoted if they look like strings, + # but usually the converter handles the quoting of string literals. + args_str = ", ".join(self.args) + return f"{prefix}{self.functor}({args_str})" + + +@dataclass +class GoalLiteral: + name: str + + def __str__(self): + return f"!{self.name}" + + +@dataclass +class ActionLiteral: + """ + Represents a step in a plan body. + Example: .say("Hello") or !achieve_goal + """ + + code: str + + def __str__(self): + return self.code + + +@dataclass +class BinaryOp: + """ + Represents logical operations. + Example: (A & B) | C + """ + + left: "Expression | str" + operator: typing.Literal["&", "|"] + right: "Expression | str" + + def __str__(self): + l_str = str(self.left) + r_str = str(self.right) + + if isinstance(self.left, BinaryOp): + l_str = f"({l_str})" + if isinstance(self.right, BinaryOp): + r_str = f"({r_str})" + + return f"{l_str} {self.operator} {r_str}" + + +Literal = BeliefLiteral | GoalLiteral | ActionLiteral +Expression = Literal | BinaryOp | str + + +@dataclass +class Rule: + """ + Represents an inference rule. + Example: head :- body. + """ + + head: Expression + body: Expression | None = None + + def __str__(self): + if not self.body: + return f"{self.head}." + return f"{self.head} :- {self.body}." + + +@dataclass +class Plan: + """ + Represents a plan. + Syntax: +trigger : context <- body. + """ + + trigger: BeliefLiteral | GoalLiteral + context: list[Expression] = field(default_factory=list) + body: list[ActionLiteral] = field(default_factory=list) + + def __str__(self): + # Indentation settings + INDENT = " " + ARROW = "\n <- " + COLON = "\n : " + + # Build Header + header = f"+{self.trigger}" + if self.context: + ctx_str = f" &\n{INDENT}".join(str(c) for c in self.context) + header += f"{COLON}{ctx_str}" + + # Case 1: Empty body + if not self.body: + return f"{header}." + + # Case 2: Short body (optional optimization, keeping it uniform usually better) + header += ARROW + + lines = [] + # We start the first action on the same line or next line. + # Let's put it on the next line for readability if there are multiple. + + if len(self.body) == 1: + return f"{header}{self.body[0]}." + + # First item + lines.append(f"{header}{self.body[0]};") + # Middle items + for item in self.body[1:-1]: + lines.append(f"{INDENT}{item};") + # Last item + lines.append(f"{INDENT}{self.body[-1]}.") + + return "\n".join(lines) + + +@dataclass +class AgentSpeakFile: + """ + Root element representing the entire generated file. + """ + + initial_beliefs: list[Rule] = field(default_factory=list) + inference_rules: list[Rule] = field(default_factory=list) + plans: list[Plan] = field(default_factory=list) + + def __str__(self): + sections = [] + + if self.initial_beliefs: + sections.append("// --- Initial Beliefs & Facts ---") + sections.extend(str(rule) for rule in self.initial_beliefs) + sections.append("") + + if self.inference_rules: + sections.append("// --- Inference Rules ---") + sections.extend(str(rule) for rule in self.inference_rules) + sections.append("") + + if self.plans: + sections.append("// --- Plans ---") + # Separate plans by a newline for readability + sections.extend(str(plan) + "\n" for plan in self.plans) + + return "\n".join(sections) diff --git a/src/control_backend/agents/bdi/asl_gen.py b/src/control_backend/agents/bdi/asl_gen.py new file mode 100644 index 0000000..f78108a --- /dev/null +++ b/src/control_backend/agents/bdi/asl_gen.py @@ -0,0 +1,295 @@ +from functools import singledispatchmethod + +from slugify import slugify + +# Import the AST we defined above +from control_backend.agents.bdi.asl_ast import ( + ActionLiteral, + AgentSpeakFile, + BeliefLiteral, + BinaryOp, + Expression, + GoalLiteral, + Plan, + Rule, +) +from control_backend.agents.bdi.bdi_program_manager import test_program + +# Import your Pydantic models (adjust import based on your file structure) +from control_backend.schemas.program import ( + Belief, + ConditionalNorm, + GestureAction, + Goal, + InferredBelief, + KeywordBelief, + LLMAction, + LogicalOperator, + Phase, + Program, + ProgramElement, + SemanticBelief, + SpeechAction, +) + + +def do_things(): + print(AgentSpeakGenerator().generate(test_program)) + + +class AgentSpeakGenerator: + """ + Converts a Pydantic Program behavior model into an AgentSpeak(L) AST, + then renders it to a string. + """ + + def generate(self, program: Program) -> str: + asl = AgentSpeakFile() + + self._generate_startup(program, asl) + + for i, phase in enumerate(program.phases): + next_phase = program.phases[i + 1] if i < len(program.phases) - 1 else None + + self._generate_phase_flow(phase, next_phase, asl) + + self._generate_norms(phase, asl) + + self._generate_goals(phase, asl) + + self._generate_triggers(phase, asl) + + return str(asl) + + # --- Section: Startup & Phase Management --- + + def _generate_startup(self, program: Program, asl: AgentSpeakFile): + if not program.phases: + return + + # Initial belief: phase(start). + asl.initial_beliefs.append(Rule(head=BeliefLiteral("phase", ["start"]))) + + # Startup plan: +started : phase(start) <- -+phase(first_id). + asl.plans.append( + Plan( + trigger=BeliefLiteral("started"), + context=[BeliefLiteral("phase", ["start"])], + body=[ActionLiteral("!transition_phase")], + ) + ) + + def _generate_phase_flow(self, phase: Phase, next_phase: Phase | None, asl: AgentSpeakFile): + """Generates the main loop listener and the transition logic for this phase.""" + + # +user_said(Message) : phase(ID) <- !goal1; !goal2; !transition_phase. + goal_actions = [ActionLiteral(f"!{self._slugify(g)}") for g in phase.goals] + goal_actions.append(ActionLiteral("!transition_phase")) + + asl.plans.append( + Plan( + trigger=BeliefLiteral("user_said", ["Message"]), + context=[BeliefLiteral("phase", [str(phase.id)])], + body=goal_actions, + ) + ) + + # +!transition_phase : phase(ID) <- -+phase(NEXT_ID). + next_id = next_phase.id if next_phase else "end" + + asl.plans.append( + Plan( + trigger=GoalLiteral("transition_phase"), + context=[BeliefLiteral("phase", [str(phase.id)])], + body=[ActionLiteral(f"-+phase({next_id})")], + ) + ) + + # --- Section: Norms & Beliefs --- + + def _generate_norms(self, phase: Phase, asl: AgentSpeakFile): + for norm in phase.norms: + norm_slug = f'"{norm.norm}"' + head = BeliefLiteral("norm", [norm_slug]) + + # Base context is the phase + phase_lit = BeliefLiteral("phase", [str(phase.id)]) + + if isinstance(norm, ConditionalNorm): + self._ensure_belief_inference(norm.condition, asl) + + condition_expr = self._belief_to_expr(norm.condition) + body = BinaryOp(phase_lit, "&", condition_expr) + else: + body = phase_lit + + asl.inference_rules.append(Rule(head=head, body=body)) + + def _ensure_belief_inference(self, belief: Belief, asl: AgentSpeakFile): + """ + Recursively adds rules to infer beliefs. + Checks strictly to avoid duplicates if necessary, + though ASL engines often handle redefinition or we can use a set to track processed IDs. + """ + if isinstance(belief, KeywordBelief): + # Rule: keyword_said("word") :- user_said(M) & .substring(M, "word", P) & P >= 0. + kwd_slug = f'"{belief.keyword}"' + head = BeliefLiteral("keyword_said", [kwd_slug]) + + # Avoid duplicates + if any(str(r.head) == str(head) for r in asl.inference_rules): + return + + body = BinaryOp( + BeliefLiteral("user_said", ["Message"]), + "&", + BinaryOp(f".substring(Message, {kwd_slug}, Pos)", "&", "Pos >= 0"), + ) + + asl.inference_rules.append(Rule(head=head, body=body)) + + elif isinstance(belief, InferredBelief): + self._ensure_belief_inference(belief.left, asl) + self._ensure_belief_inference(belief.right, asl) + + slug = self._slugify(belief) + head = BeliefLiteral(slug) + + if any(str(r.head) == str(head) for r in asl.inference_rules): + return + + op_char = "&" if belief.operator == LogicalOperator.AND else "|" + body = BinaryOp( + self._belief_to_expr(belief.left), op_char, self._belief_to_expr(belief.right) + ) + asl.inference_rules.append(Rule(head=head, body=body)) + + def _belief_to_expr(self, belief: Belief) -> Expression: + if isinstance(belief, KeywordBelief): + return BeliefLiteral("keyword_said", [f'"{belief.keyword}"']) + else: + return BeliefLiteral(self._slugify(belief)) + + # --- Section: Goals --- + + def _generate_goals(self, phase: Phase, asl: AgentSpeakFile): + previous_goal: Goal | None = None + for goal in phase.goals: + self._generate_goal_plan_recursive(goal, str(phase.id), previous_goal, asl) + previous_goal = goal + + def _generate_goal_plan_recursive( + self, goal: Goal, phase_id: str, previous_goal: Goal | None, asl: AgentSpeakFile + ): + goal_slug = self._slugify(goal) + + # phase(ID) & not responded_this_turn & not achieved_goal + context = [ + BeliefLiteral("phase", [phase_id]), + BeliefLiteral("responded_this_turn", negated=True), + BeliefLiteral(f"achieved_{goal_slug}", negated=True), + ] + + if previous_goal: + prev_slug = self._slugify(previous_goal) + context.append(BeliefLiteral(f"achieved_{prev_slug}")) + + body_actions = [] + sub_goals_to_process = [] + + for step in goal.plan.steps: + if isinstance(step, Goal): + sub_slug = self._slugify(step) + body_actions.append(ActionLiteral(f"!{sub_slug}")) + sub_goals_to_process.append(step) + elif isinstance(step, SpeechAction): + body_actions.append(ActionLiteral(f'.say("{step.text}")')) + elif isinstance(step, GestureAction): + body_actions.append(ActionLiteral(f'.gesture("{step.gesture}")')) + elif isinstance(step, LLMAction): + body_actions.append(ActionLiteral(f'!generate_response_with_goal("{step.goal}")')) + + # Mark achievement + if not goal.can_fail: + body_actions.append(ActionLiteral(f"+achieved_{goal_slug}")) + + asl.plans.append(Plan(trigger=GoalLiteral(goal_slug), context=context, body=body_actions)) + + prev_sub = None + for sub_goal in sub_goals_to_process: + self._generate_goal_plan_recursive(sub_goal, phase_id, prev_sub, asl) + prev_sub = sub_goal + + # --- Section: Triggers --- + + def _generate_triggers(self, phase: Phase, asl: AgentSpeakFile): + for trigger in phase.triggers: + self._ensure_belief_inference(trigger.condition, asl) + + trigger_belief_slug = self._belief_to_expr(trigger.condition) + + body_actions = [] + sub_goals = [] + + for step in trigger.plan.steps: + if isinstance(step, Goal): + sub_slug = self._slugify(step) + body_actions.append(ActionLiteral(f"!{sub_slug}")) + sub_goals.append(step) + elif isinstance(step, SpeechAction): + body_actions.append(ActionLiteral(f'.say("{step.text}")')) + elif isinstance(step, GestureAction): + body_actions.append( + ActionLiteral(f'.gesture("{step.gesture.type}", "{step.gesture.name}")') + ) + elif isinstance(step, LLMAction): + body_actions.append( + ActionLiteral(f'!generate_response_with_goal("{step.goal}")') + ) + + asl.plans.append( + Plan( + trigger=BeliefLiteral(trigger_belief_slug), + context=[BeliefLiteral("phase", [str(phase.id)])], + body=body_actions, + ) + ) + + # Recurse for triggered goals + prev_sub = None + for sub_goal in sub_goals: + self._generate_goal_plan_recursive(sub_goal, str(phase.id), prev_sub, asl) + prev_sub = sub_goal + + # --- Helpers --- + + @singledispatchmethod + def _slugify(self, element: ProgramElement) -> str: + if element.name: + raise NotImplementedError("Cannot slugify this element.") + return self._slugify_str(element.name) + + @_slugify.register + def _(self, goal: Goal) -> str: + if goal.name: + return self._slugify_str(goal.name) + return f"goal_{goal.id}" + + @_slugify.register + def _(self, kwb: KeywordBelief) -> str: + return f"keyword_said({kwb.keyword})" + + @_slugify.register + def _(self, sb: SemanticBelief) -> str: + return self._slugify_str(sb.description) + + @_slugify.register + def _(self, ib: InferredBelief) -> str: + return self._slugify_str(ib.name) + + def _slugify_str(self, text: str) -> str: + return slugify(text, separator="_", stopwords=["a", "an", "the", "we", "you", "I"]) + + +if __name__ == "__main__": + do_things() diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 11c0b00..9925cfb 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -1,3 +1,4 @@ +import uuid from collections.abc import Iterable import zmq @@ -32,53 +33,72 @@ test_program = Program( phases=[ Phase( norms=[ - BasicNorm(norm="Talk like a pirate"), + BasicNorm(norm="Talk like a pirate", id=uuid.uuid4()), ConditionalNorm( condition=InferredBelief( - left=KeywordBelief(keyword="Arr"), - right=SemanticBelief(description="testing", name="semantic belief"), + left=KeywordBelief(keyword="Arr", id=uuid.uuid4()), + right=SemanticBelief( + description="testing", name="semantic belief", id=uuid.uuid4() + ), operator=LogicalOperator.OR, name="Talking to a pirate", + id=uuid.uuid4(), ), norm="Use nautical terms", + id=uuid.uuid4(), ), ConditionalNorm( condition=SemanticBelief( - description="We are talking to a child", name="talking to child" + description="We are talking to a child", + name="talking to child", + id=uuid.uuid4(), ), norm="Do not use cuss words", + id=uuid.uuid4(), ), ], triggers=[ Trigger( condition=InferredBelief( - left=KeywordBelief(keyword="key"), + left=KeywordBelief(keyword="key", id=uuid.uuid4()), right=InferredBelief( - left=KeywordBelief(keyword="key2"), + left=KeywordBelief(keyword="key2", id=uuid.uuid4()), right=SemanticBelief( - description="Decode this", name="semantic belief 2" + description="Decode this", name="semantic belief 2", id=uuid.uuid4() ), operator=LogicalOperator.OR, name="test trigger inferred inner", + id=uuid.uuid4(), ), operator=LogicalOperator.OR, name="test trigger inferred outer", + id=uuid.uuid4(), ), plan=Plan( steps=[ - SpeechAction(text="Testing trigger"), + SpeechAction(text="Testing trigger", id=uuid.uuid4()), Goal( name="Testing trigger", - plan=Plan(steps=[LLMAction(goal="Do something")]), + plan=Plan( + steps=[LLMAction(goal="Do something", id=uuid.uuid4())], + id=uuid.uuid4(), + ), + id=uuid.uuid4(), ), - ] + ], + id=uuid.uuid4(), ), + id=uuid.uuid4(), ) ], goals=[ Goal( name="Determine user age", - plan=Plan(steps=[LLMAction(goal="Determine the age of the user.")]), + plan=Plan( + steps=[LLMAction(goal="Determine the age of the user.", id=uuid.uuid4())], + id=uuid.uuid4(), + ), + id=uuid.uuid4(), ), Goal( name="Find the user's name", @@ -86,38 +106,62 @@ test_program = Program( steps=[ Goal( name="Greet the user", - plan=Plan(steps=[LLMAction(goal="Greet the user.")]), + plan=Plan( + steps=[LLMAction(goal="Greet the user.", id=uuid.uuid4())], + id=uuid.uuid4(), + ), can_fail=False, + id=uuid.uuid4(), ), Goal( name="Ask for name", - plan=Plan(steps=[LLMAction(goal="Obtain the user's name.")]), + plan=Plan( + steps=[ + LLMAction(goal="Obtain the user's name.", id=uuid.uuid4()) + ], + id=uuid.uuid4(), + ), + id=uuid.uuid4(), ), - ] + ], + id=uuid.uuid4(), ), + id=uuid.uuid4(), + ), + Goal( + name="Tell a joke", + plan=Plan( + steps=[LLMAction(goal="Tell a joke.", id=uuid.uuid4())], id=uuid.uuid4() + ), + id=uuid.uuid4(), ), - Goal(name="Tell a joke", plan=Plan(steps=[LLMAction(goal="Tell a joke.")])), ], - id=1, + id=uuid.uuid4(), ), Phase( - id=2, + id=uuid.uuid4(), norms=[ - BasicNorm(norm="Use very gentle speech."), + BasicNorm(norm="Use very gentle speech.", id=uuid.uuid4()), ConditionalNorm( condition=SemanticBelief( - description="We are talking to a child", name="talking to child" + description="We are talking to a child", + name="talking to child", + id=uuid.uuid4(), ), norm="Do not use cuss words", + id=uuid.uuid4(), ), ], triggers=[ Trigger( condition=InferredBelief( - left=KeywordBelief(keyword="help"), - right=SemanticBelief(description="User is stuck", name="stuck"), + left=KeywordBelief(keyword="help", id=uuid.uuid4()), + right=SemanticBelief( + description="User is stuck", name="stuck", id=uuid.uuid4() + ), operator=LogicalOperator.OR, name="help_or_stuck", + id=uuid.uuid4(), ), plan=Plan( steps=[ @@ -127,13 +171,18 @@ test_program = Program( steps=[ LLMAction( goal="Provide a step-by-step path to " - "resolve the user's issue." + "resolve the user's issue.", + id=uuid.uuid4(), ) - ] + ], + id=uuid.uuid4(), ), + id=uuid.uuid4(), ), - ] + ], + id=uuid.uuid4(), ), + id=uuid.uuid4(), ), ], goals=[ @@ -143,20 +192,38 @@ test_program = Program( steps=[ LLMAction( goal="Ask 1-2 targeted questions to clarify the " - "user's intent, then proceed." + "user's intent, then proceed.", + id=uuid.uuid4(), ) - ] + ], + id=uuid.uuid4(), ), + id=uuid.uuid4(), ), Goal( name="Provide solution", plan=Plan( - steps=[LLMAction(goal="Deliver a solution to complete the user's goal.")] + steps=[ + LLMAction( + goal="Deliver a solution to complete the user's goal.", + id=uuid.uuid4(), + ) + ], + id=uuid.uuid4(), ), + id=uuid.uuid4(), ), Goal( name="Summarize next steps", - plan=Plan(steps=[LLMAction(goal="Summarize what the user should do next.")]), + plan=Plan( + steps=[ + LLMAction( + goal="Summarize what the user should do next.", id=uuid.uuid4() + ) + ], + id=uuid.uuid4(), + ), + id=uuid.uuid4(), ), ], ), @@ -198,10 +265,16 @@ class AgentSpeakGenerator: return "\n".join(lines) def _generate_initial_beliefs(self, program: Program) -> Iterable[str]: - yield "// --- Initial beliefs ---" + yield "// --- Initial beliefs and agent startup ---" yield "phase(start)." + yield "" + + yield "+started" + yield f"{self.colon_prefix}phase(start)" + yield f"{self.arrow_prefix}phase({program.phases[0].id if program.phases else 'end'})." + yield from ["", ""] def _generate_basic_flow(self, program: Program) -> Iterable[str]: diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 7c73a6a..529a23d 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -1,4 +1,5 @@ from enum import Enum +from typing import Literal from pydantic import UUID4, BaseModel @@ -133,9 +134,17 @@ class SpeechAction(ProgramElement): text: str -# TODO: gestures -class Gesture(Enum): - RAISE_HAND = "RAISE_HAND" +class Gesture(BaseModel): + """ + Represents a gesture to be performed. Can be either a single gesture, + or a random gesture from a category (tag). + + :ivar type: The type of the gesture, "tag" or "single". + :ivar name: The name of the single gesture or tag. + """ + + type: Literal["tag", "single"] + name: str class GestureAction(ProgramElement): From 28262eb27e2650a8fe959a8905a6c9ffb659b8d9 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 17 Dec 2025 16:20:37 +0100 Subject: [PATCH 09/90] fix: default case for plans ref: N25B-376 --- src/control_backend/agents/bdi/asl_gen.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/control_backend/agents/bdi/asl_gen.py b/src/control_backend/agents/bdi/asl_gen.py index f78108a..7d0fa77 100644 --- a/src/control_backend/agents/bdi/asl_gen.py +++ b/src/control_backend/agents/bdi/asl_gen.py @@ -214,6 +214,9 @@ class AgentSpeakGenerator: body_actions.append(ActionLiteral(f"+achieved_{goal_slug}")) asl.plans.append(Plan(trigger=GoalLiteral(goal_slug), context=context, body=body_actions)) + asl.plans.append( + Plan(trigger=GoalLiteral(goal_slug), context=[], body=[ActionLiteral("true")]) + ) prev_sub = None for sub_goal in sub_goals_to_process: From f91cec670854a28ce42ea1a3b39a7806e73188c1 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Thu, 18 Dec 2025 11:50:16 +0100 Subject: [PATCH 10/90] fix: things in AgentSpeak, add custom actions ref: N25B-376 --- src/control_backend/agents/bdi/asl_gen.py | 105 ++++++++++++++---- .../agents/bdi/bdi_core_agent.py | 82 ++++++++++++-- 2 files changed, 153 insertions(+), 34 deletions(-) diff --git a/src/control_backend/agents/bdi/asl_gen.py b/src/control_backend/agents/bdi/asl_gen.py index 7d0fa77..845b4e3 100644 --- a/src/control_backend/agents/bdi/asl_gen.py +++ b/src/control_backend/agents/bdi/asl_gen.py @@ -1,7 +1,11 @@ +import asyncio +import time from functools import singledispatchmethod from slugify import slugify +from control_backend.agents.bdi import BDICoreAgent + # Import the AST we defined above from control_backend.agents.bdi.asl_ast import ( ActionLiteral, @@ -33,8 +37,20 @@ from control_backend.schemas.program import ( ) -def do_things(): - print(AgentSpeakGenerator().generate(test_program)) +async def do_things(): + res = input("Wanna generate") + if res == "y": + program = AgentSpeakGenerator().generate(test_program) + filename = f"{int(time.time())}.asl" + with open(filename, "w") as f: + f.write(program) + else: + # filename = "0test.asl" + filename = "1766053943.asl" + bdi_agent = BDICoreAgent("BDICoreAgent", filename) + flag = asyncio.Event() + await bdi_agent.start() + await flag.wait() class AgentSpeakGenerator: @@ -59,6 +75,8 @@ class AgentSpeakGenerator: self._generate_triggers(phase, asl) + self._generate_fallbacks(program, asl) + return str(asl) # --- Section: Startup & Phase Management --- @@ -68,14 +86,30 @@ class AgentSpeakGenerator: return # Initial belief: phase(start). - asl.initial_beliefs.append(Rule(head=BeliefLiteral("phase", ["start"]))) + asl.initial_beliefs.append(Rule(head=BeliefLiteral("phase", ['"start"']))) - # Startup plan: +started : phase(start) <- -+phase(first_id). + # Startup plan: +started : phase(start) <- -phase(start); +phase(first_id). asl.plans.append( Plan( trigger=BeliefLiteral("started"), - context=[BeliefLiteral("phase", ["start"])], - body=[ActionLiteral("!transition_phase")], + context=[BeliefLiteral("phase", ['"start"'])], + body=[ + ActionLiteral('-phase("start")'), + ActionLiteral(f'+phase("{program.phases[0].id}")'), + ], + ) + ) + + # Initial plans: + asl.plans.append( + Plan( + trigger=GoalLiteral("generate_response_with_goal(Goal)"), + context=[BeliefLiteral("user_said", ["Message"])], + body=[ + ActionLiteral("+responded_this_turn"), + ActionLiteral(".findall(Norm, norm(Norm), Norms)"), + ActionLiteral(".reply_with_goal(Message, Norms, Goal)"), + ], ) ) @@ -83,25 +117,33 @@ class AgentSpeakGenerator: """Generates the main loop listener and the transition logic for this phase.""" # +user_said(Message) : phase(ID) <- !goal1; !goal2; !transition_phase. - goal_actions = [ActionLiteral(f"!{self._slugify(g)}") for g in phase.goals] + goal_actions = [ActionLiteral("-responded_this_turn")] + goal_actions += [ActionLiteral(f"!{self._slugify(g)}") for g in phase.goals] goal_actions.append(ActionLiteral("!transition_phase")) asl.plans.append( Plan( trigger=BeliefLiteral("user_said", ["Message"]), - context=[BeliefLiteral("phase", [str(phase.id)])], + context=[BeliefLiteral("phase", [f'"{phase.id}"'])], body=goal_actions, ) ) - # +!transition_phase : phase(ID) <- -+phase(NEXT_ID). - next_id = next_phase.id if next_phase else "end" + # +!transition_phase : phase(ID) <- -phase(ID); +(NEXT_ID). + next_id = str(next_phase.id) if next_phase else "end" + + transition_context = [BeliefLiteral("phase", [f'"{phase.id}"'])] + if phase.goals: + transition_context.append(BeliefLiteral(f"achieved_{self._slugify(phase.goals[-1])}")) asl.plans.append( Plan( trigger=GoalLiteral("transition_phase"), - context=[BeliefLiteral("phase", [str(phase.id)])], - body=[ActionLiteral(f"-+phase({next_id})")], + context=transition_context, + body=[ + ActionLiteral(f'-phase("{phase.id}")'), + ActionLiteral(f'+phase("{next_id}")'), + ], ) ) @@ -113,7 +155,7 @@ class AgentSpeakGenerator: head = BeliefLiteral("norm", [norm_slug]) # Base context is the phase - phase_lit = BeliefLiteral("phase", [str(phase.id)]) + phase_lit = BeliefLiteral("phase", [f'"{phase.id}"']) if isinstance(norm, ConditionalNorm): self._ensure_belief_inference(norm.condition, asl) @@ -132,7 +174,7 @@ class AgentSpeakGenerator: though ASL engines often handle redefinition or we can use a set to track processed IDs. """ if isinstance(belief, KeywordBelief): - # Rule: keyword_said("word") :- user_said(M) & .substring(M, "word", P) & P >= 0. + # Rule: keyword_said("word") :- user_said(M) & .substring("word", M, P) & P >= 0. kwd_slug = f'"{belief.keyword}"' head = BeliefLiteral("keyword_said", [kwd_slug]) @@ -143,7 +185,7 @@ class AgentSpeakGenerator: body = BinaryOp( BeliefLiteral("user_said", ["Message"]), "&", - BinaryOp(f".substring(Message, {kwd_slug}, Pos)", "&", "Pos >= 0"), + BinaryOp(f".substring({kwd_slug}, Message, Pos)", "&", "Pos >= 0"), ) asl.inference_rules.append(Rule(head=head, body=body)) @@ -185,7 +227,7 @@ class AgentSpeakGenerator: # phase(ID) & not responded_this_turn & not achieved_goal context = [ - BeliefLiteral("phase", [phase_id]), + BeliefLiteral("phase", [f'"{phase_id}"']), BeliefLiteral("responded_this_turn", negated=True), BeliefLiteral(f"achieved_{goal_slug}", negated=True), ] @@ -214,9 +256,6 @@ class AgentSpeakGenerator: body_actions.append(ActionLiteral(f"+achieved_{goal_slug}")) asl.plans.append(Plan(trigger=GoalLiteral(goal_slug), context=context, body=body_actions)) - asl.plans.append( - Plan(trigger=GoalLiteral(goal_slug), context=[], body=[ActionLiteral("true")]) - ) prev_sub = None for sub_goal in sub_goals_to_process: @@ -253,7 +292,7 @@ class AgentSpeakGenerator: asl.plans.append( Plan( trigger=BeliefLiteral(trigger_belief_slug), - context=[BeliefLiteral("phase", [str(phase.id)])], + context=[BeliefLiteral("phase", [f'"{phase.id}"'])], body=body_actions, ) ) @@ -264,6 +303,28 @@ class AgentSpeakGenerator: self._generate_goal_plan_recursive(sub_goal, str(phase.id), prev_sub, asl) prev_sub = sub_goal + # --- Section: Fallbacks --- + + def _generate_fallbacks(self, program: Program, asl: AgentSpeakFile): + for phase in program.phases: + for goal in phase.goals: + self._generate_goal_fallbacks_recursive(goal, asl) + + asl.plans.append( + Plan(trigger=GoalLiteral("transition_phase"), context=[], body=[ActionLiteral("true")]) + ) + + def _generate_goal_fallbacks_recursive(self, goal: Goal, asl: AgentSpeakFile): + goal_slug = self._slugify(goal) + asl.plans.append( + Plan(trigger=GoalLiteral(goal_slug), context=[], body=[ActionLiteral("true")]) + ) + + for step in goal.plan.steps: + if not isinstance(step, Goal): + continue + self._generate_goal_fallbacks_recursive(step, asl) + # --- Helpers --- @singledispatchmethod @@ -276,7 +337,7 @@ class AgentSpeakGenerator: def _(self, goal: Goal) -> str: if goal.name: return self._slugify_str(goal.name) - return f"goal_{goal.id}" + return f"goal_{goal.id.hex}" @_slugify.register def _(self, kwb: KeywordBelief) -> str: @@ -295,4 +356,4 @@ class AgentSpeakGenerator: if __name__ == "__main__": - do_things() + asyncio.run(do_things()) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index f056e09..9408ff8 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -160,7 +160,7 @@ class BDICoreAgent(BaseAgent): self._remove_all_with_name(belief.name) self._add_belief(belief.name, belief.arguments) - def _add_belief(self, name: str, args: Iterable[str] = []): + def _add_belief(self, name: str, args: list[str] = None): """ Add a single belief to the BDI agent. @@ -168,9 +168,12 @@ class BDICoreAgent(BaseAgent): :param args: Arguments for the belief. """ # new_args = (agentspeak.Literal(arg) for arg in args) # TODO: Eventually support multiple - merged_args = DELIMITER.join(arg for arg in args) - new_args = (agentspeak.Literal(merged_args),) - term = agentspeak.Literal(name, new_args) + if args: + merged_args = DELIMITER.join(arg for arg in args) + new_args = (agentspeak.Literal(merged_args),) + term = agentspeak.Literal(name, new_args) + else: + term = agentspeak.Literal(name) self.bdi_agent.call( agentspeak.Trigger.addition, @@ -238,8 +241,7 @@ class BDICoreAgent(BaseAgent): @self.actions.add(".reply", 3) def _reply(agent: "BDICoreAgent", term, intention): """ - Sends text to the LLM (AgentSpeak action). - Example: .reply("Hello LLM!", "Some norm", "Some goal") + Let the LLM generate a response to a user's utterance with the current norms and goals. """ message_text = agentspeak.grounded(term.args[0], intention.scope) norms = agentspeak.grounded(term.args[1], intention.scope) @@ -252,15 +254,71 @@ class BDICoreAgent(BaseAgent): asyncio.create_task(self._send_to_llm(str(message_text), str(norms), str(goals))) yield - async def _send_to_llm(self, text: str, norms: str = None, goals: str = None): + @self.actions.add(".reply_with_goal", 3) + def _reply_with_goal(agent: "BDICoreAgent", term, intention): + """ + Let the LLM generate a response to a user's utterance with the current norms and a + specific goal. + """ + message_text = agentspeak.grounded(term.args[0], intention.scope) + norms = agentspeak.grounded(term.args[1], intention.scope) + goal = agentspeak.grounded(term.args[2], intention.scope) + + self.logger.debug( + '"reply_with_goal" action called with message=%s, norms=%s, goal=%s', + message_text, + norms, + goal, + ) + # asyncio.create_task(self._send_to_llm(str(message_text), norms, str(goal))) + yield + + @self.actions.add(".say", 1) + def _say(agent: "BDICoreAgent", term, intention): + """ + Make the robot say the given text instantly. + """ + message_text = agentspeak.grounded(term.args[0], intention.scope) + + self.logger.debug('"say" action called with text=%s', message_text) + + # speech_command = SpeechCommand(data=message_text) + # speech_message = InternalMessage( + # to=settings.agent_settings.robot_speech_name, + # sender=settings.agent_settings.bdi_core_name, + # body=speech_command.model_dump_json(), + # ) + # asyncio.create_task(agent.send(speech_message)) + yield + + @self.actions.add(".gesture", 2) + def _gesture(agent: "BDICoreAgent", term, intention): + """ + Make the robot perform the given gesture instantly. + """ + gesture_type = agentspeak.grounded(term.args[0], intention.scope) + gesture_name = agentspeak.grounded(term.args[1], intention.scope) + + self.logger.debug( + '"gesture" action called with type=%s, name=%s', + gesture_type, + gesture_name, + ) + + # gesture = Gesture(type=gesture_type, name=gesture_name) + # gesture_message = InternalMessage( + # to=settings.agent_settings.robot_gesture_name, + # sender=settings.agent_settings.bdi_core_name, + # body=gesture.model_dump_json(), + # ) + # asyncio.create_task(agent.send(gesture_message)) + yield + + async def _send_to_llm(self, text: str, norms: str, goals: str): """ Sends a text query to the LLM agent asynchronously. """ - prompt = LLMPromptMessage( - text=text, - norms=norms.split("\n") if norms else [], - goals=goals.split("\n") if norms else [], - ) + prompt = LLMPromptMessage(text=text, norms=norms.split("\n"), goals=goals.split("\n")) msg = InternalMessage( to=settings.agent_settings.llm_name, sender=self.name, From 756e1f0dc5b59b2e8584b29c98a6ee28737e3227 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Thu, 18 Dec 2025 14:33:42 +0100 Subject: [PATCH 11/90] feat: persistent rules and stuff So ugly ref: N25B-376 --- src/control_backend/agents/bdi/asl_ast.py | 35 ++++- src/control_backend/agents/bdi/asl_gen.py | 146 +++++++++++++----- .../agents/bdi/bdi_core_agent.py | 7 +- 3 files changed, 143 insertions(+), 45 deletions(-) diff --git a/src/control_backend/agents/bdi/asl_ast.py b/src/control_backend/agents/bdi/asl_ast.py index 6543b63..104570b 100644 --- a/src/control_backend/agents/bdi/asl_ast.py +++ b/src/control_backend/agents/bdi/asl_ast.py @@ -93,6 +93,33 @@ class Rule: return f"{self.head} :- {self.body}." +@dataclass +class PersistentRule: + """ + Represents an inference rule, where the inferred belief is persistent when formed. + """ + + head: Expression + body: Expression + + def __str__(self): + if not self.body: + raise Exception("Rule without body should not be persistent.") + + lines = [] + + if isinstance(self.body, BinaryOp): + lines.append(f"+{self.body.left}") + if self.body.operator == "&": + lines.append(f" : {self.body.right}") + lines.append(f" <- +{self.head}.") + if self.body.operator == "|": + lines.append(f"+{self.body.right}") + lines.append(f" <- +{self.head}.") + + return "\n".join(lines) + + @dataclass class Plan: """ @@ -148,7 +175,7 @@ class AgentSpeakFile: """ initial_beliefs: list[Rule] = field(default_factory=list) - inference_rules: list[Rule] = field(default_factory=list) + inference_rules: list[Rule | PersistentRule] = field(default_factory=list) plans: list[Plan] = field(default_factory=list) def __str__(self): @@ -161,7 +188,11 @@ class AgentSpeakFile: if self.inference_rules: sections.append("// --- Inference Rules ---") - sections.extend(str(rule) for rule in self.inference_rules) + sections.extend(str(rule) for rule in self.inference_rules if isinstance(rule, Rule)) + sections.append("") + sections.extend( + str(rule) for rule in self.inference_rules if isinstance(rule, PersistentRule) + ) sections.append("") if self.plans: diff --git a/src/control_backend/agents/bdi/asl_gen.py b/src/control_backend/agents/bdi/asl_gen.py index 845b4e3..8233a36 100644 --- a/src/control_backend/agents/bdi/asl_gen.py +++ b/src/control_backend/agents/bdi/asl_gen.py @@ -5,8 +5,6 @@ from functools import singledispatchmethod from slugify import slugify from control_backend.agents.bdi import BDICoreAgent - -# Import the AST we defined above from control_backend.agents.bdi.asl_ast import ( ActionLiteral, AgentSpeakFile, @@ -14,13 +12,13 @@ from control_backend.agents.bdi.asl_ast import ( BinaryOp, Expression, GoalLiteral, + PersistentRule, Plan, Rule, ) from control_backend.agents.bdi.bdi_program_manager import test_program - -# Import your Pydantic models (adjust import based on your file structure) from control_backend.schemas.program import ( + BasicBelief, Belief, ConditionalNorm, GestureAction, @@ -46,13 +44,17 @@ async def do_things(): f.write(program) else: # filename = "0test.asl" - filename = "1766053943.asl" + filename = "1766062491.asl" bdi_agent = BDICoreAgent("BDICoreAgent", filename) flag = asyncio.Event() await bdi_agent.start() await flag.wait() +def do_other_things(): + print(AgentSpeakGenerator().generate(test_program)) + + class AgentSpeakGenerator: """ Converts a Pydantic Program behavior model into an AgentSpeak(L) AST, @@ -118,6 +120,10 @@ class AgentSpeakGenerator: # +user_said(Message) : phase(ID) <- !goal1; !goal2; !transition_phase. goal_actions = [ActionLiteral("-responded_this_turn")] + goal_actions += [ + ActionLiteral(f"!check_{self._slugify_str(keyword)}") + for keyword in self._get_keyword_conditionals(phase) + ] goal_actions += [ActionLiteral(f"!{self._slugify(g)}") for g in phase.goals] goal_actions.append(ActionLiteral("!transition_phase")) @@ -143,10 +149,20 @@ class AgentSpeakGenerator: body=[ ActionLiteral(f'-phase("{phase.id}")'), ActionLiteral(f'+phase("{next_id}")'), + ActionLiteral("user_said(Anything)"), + ActionLiteral("-+user_said(Anything)"), ], ) ) + def _get_keyword_conditionals(self, phase: Phase) -> list[str]: + res = [] + for belief in self._extract_basic_beliefs_from_phase(phase): + if isinstance(belief, KeywordBelief): + res.append(belief.keyword) + + return res + # --- Section: Norms & Beliefs --- def _generate_norms(self, phase: Phase, asl: AgentSpeakFile): @@ -174,21 +190,22 @@ class AgentSpeakGenerator: though ASL engines often handle redefinition or we can use a set to track processed IDs. """ if isinstance(belief, KeywordBelief): - # Rule: keyword_said("word") :- user_said(M) & .substring("word", M, P) & P >= 0. - kwd_slug = f'"{belief.keyword}"' - head = BeliefLiteral("keyword_said", [kwd_slug]) - - # Avoid duplicates - if any(str(r.head) == str(head) for r in asl.inference_rules): - return - - body = BinaryOp( - BeliefLiteral("user_said", ["Message"]), - "&", - BinaryOp(f".substring({kwd_slug}, Message, Pos)", "&", "Pos >= 0"), - ) - - asl.inference_rules.append(Rule(head=head, body=body)) + pass + # # Rule: keyword_said("word") :- user_said(M) & .substring("word", M, P) & P >= 0. + # kwd_slug = f'"{belief.keyword}"' + # head = BeliefLiteral("keyword_said", [kwd_slug]) + # + # # Avoid duplicates + # if any(str(r.head) == str(head) for r in asl.inference_rules): + # return + # + # body = BinaryOp( + # BeliefLiteral("user_said", ["Message"]), + # "&", + # BinaryOp(f".substring({kwd_slug}, Message, Pos)", "&", "Pos >= 0"), + # ) + # + # asl.inference_rules.append(Rule(head=head, body=body)) elif isinstance(belief, InferredBelief): self._ensure_belief_inference(belief.left, asl) @@ -204,7 +221,7 @@ class AgentSpeakGenerator: body = BinaryOp( self._belief_to_expr(belief.left), op_char, self._belief_to_expr(belief.right) ) - asl.inference_rules.append(Rule(head=head, body=body)) + asl.inference_rules.append(PersistentRule(head=head, body=body)) def _belief_to_expr(self, belief: Belief) -> Expression: if isinstance(belief, KeywordBelief): @@ -221,17 +238,26 @@ class AgentSpeakGenerator: previous_goal = goal def _generate_goal_plan_recursive( - self, goal: Goal, phase_id: str, previous_goal: Goal | None, asl: AgentSpeakFile + self, + goal: Goal, + phase_id: str, + previous_goal: Goal | None, + asl: AgentSpeakFile, + responded_needed: bool = True, + can_fail: bool = True, ): goal_slug = self._slugify(goal) # phase(ID) & not responded_this_turn & not achieved_goal context = [ BeliefLiteral("phase", [f'"{phase_id}"']), - BeliefLiteral("responded_this_turn", negated=True), - BeliefLiteral(f"achieved_{goal_slug}", negated=True), ] + if responded_needed: + context.append(BeliefLiteral("responded_this_turn", negated=True)) + if can_fail: + context.append(BeliefLiteral(f"achieved_{goal_slug}", negated=True)) + if previous_goal: prev_slug = self._slugify(previous_goal) context.append(BeliefLiteral(f"achieved_{prev_slug}")) @@ -256,6 +282,9 @@ class AgentSpeakGenerator: body_actions.append(ActionLiteral(f"+achieved_{goal_slug}")) asl.plans.append(Plan(trigger=GoalLiteral(goal_slug), context=context, body=body_actions)) + asl.plans.append( + Plan(trigger=GoalLiteral(goal_slug), context=[], body=[ActionLiteral("true")]) + ) prev_sub = None for sub_goal in sub_goals_to_process: @@ -265,6 +294,28 @@ class AgentSpeakGenerator: # --- Section: Triggers --- def _generate_triggers(self, phase: Phase, asl: AgentSpeakFile): + for keyword in self._get_keyword_conditionals(phase): + asl.plans.append( + Plan( + trigger=GoalLiteral(f"check_{self._slugify_str(keyword)}"), + context=[ + ActionLiteral( + f'user_said(Message) & .substring("{keyword}", Message, Pos) & Pos >= 0' + ) + ], + body=[ + ActionLiteral(f'+keyword_said("{keyword}")'), + ActionLiteral(f'-keyword_said("{keyword}")'), + ], + ) + ) + asl.plans.append( + Plan( + trigger=GoalLiteral(f"check_{self._slugify_str(keyword)}"), + body=[ActionLiteral("true")], + ) + ) + for trigger in phase.triggers: self._ensure_belief_inference(trigger.condition, asl) @@ -300,31 +351,18 @@ class AgentSpeakGenerator: # Recurse for triggered goals prev_sub = None for sub_goal in sub_goals: - self._generate_goal_plan_recursive(sub_goal, str(phase.id), prev_sub, asl) + self._generate_goal_plan_recursive( + sub_goal, str(phase.id), prev_sub, asl, False, False + ) prev_sub = sub_goal # --- Section: Fallbacks --- def _generate_fallbacks(self, program: Program, asl: AgentSpeakFile): - for phase in program.phases: - for goal in phase.goals: - self._generate_goal_fallbacks_recursive(goal, asl) - asl.plans.append( Plan(trigger=GoalLiteral("transition_phase"), context=[], body=[ActionLiteral("true")]) ) - def _generate_goal_fallbacks_recursive(self, goal: Goal, asl: AgentSpeakFile): - goal_slug = self._slugify(goal) - asl.plans.append( - Plan(trigger=GoalLiteral(goal_slug), context=[], body=[ActionLiteral("true")]) - ) - - for step in goal.plan.steps: - if not isinstance(step, Goal): - continue - self._generate_goal_fallbacks_recursive(step, asl) - # --- Helpers --- @singledispatchmethod @@ -354,6 +392,34 @@ class AgentSpeakGenerator: def _slugify_str(self, text: str) -> str: return slugify(text, separator="_", stopwords=["a", "an", "the", "we", "you", "I"]) + def _extract_basic_beliefs_from_program(self, program: Program) -> list[BasicBelief]: + beliefs = [] + + for phase in program.phases: + beliefs.extend(self._extract_basic_beliefs_from_phase(phase)) + + return beliefs + + def _extract_basic_beliefs_from_phase(self, phase: Phase) -> list[BasicBelief]: + beliefs = [] + + for norm in phase.norms: + if isinstance(norm, ConditionalNorm): + beliefs += self._extract_basic_beliefs_from_belief(norm.condition) + + for trigger in phase.triggers: + beliefs += self._extract_basic_beliefs_from_belief(trigger.condition) + + return beliefs + + def _extract_basic_beliefs_from_belief(self, belief: Belief) -> list[BasicBelief]: + if isinstance(belief, InferredBelief): + return self._extract_basic_beliefs_from_belief( + belief.left + ) + self._extract_basic_beliefs_from_belief(belief.right) + return [belief] + if __name__ == "__main__": asyncio.run(do_things()) + # do_other_things()y diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 9408ff8..8ff271c 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -89,9 +89,9 @@ class BDICoreAgent(BaseAgent): the agent has deferred intentions (deadlines). """ while self._running: - await ( - self._wake_bdi_loop.wait() - ) # gets set whenever there's an update to the belief base + # await ( + # self._wake_bdi_loop.wait() + # ) # gets set whenever there's an update to the belief base # Agent knows when it's expected to have to do its next thing maybe_more_work = True @@ -168,6 +168,7 @@ class BDICoreAgent(BaseAgent): :param args: Arguments for the belief. """ # new_args = (agentspeak.Literal(arg) for arg in args) # TODO: Eventually support multiple + args = args or [] if args: merged_args = DELIMITER.join(arg for arg in args) new_args = (agentspeak.Literal(merged_args),) From 33501093a1ea467acb243263b12ea027c4ff8447 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:09:58 +0100 Subject: [PATCH 12/90] feat: extract semantic beliefs from conversation ref: N25B-380 --- .../agents/bdi/text_belief_extractor_agent.py | 325 ++++++++++++++++-- src/control_backend/agents/llm/llm_agent.py | 22 +- src/control_backend/core/config.py | 11 + src/control_backend/schemas/chat_history.py | 10 + src/control_backend/schemas/program.py | 203 +++++++++-- 5 files changed, 508 insertions(+), 63 deletions(-) create mode 100644 src/control_backend/schemas/chat_history.py diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index 0f2db01..0324573 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -1,8 +1,23 @@ +import asyncio import json +import httpx +from pydantic import ValidationError +from slugify import slugify + from control_backend.agents.base import BaseAgent from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings +from control_backend.schemas.belief_message import Belief as InternalBelief +from control_backend.schemas.belief_message import BeliefMessage +from control_backend.schemas.chat_history import ChatHistory, ChatMessage +from control_backend.schemas.program import ( + Belief, + ConditionalNorm, + InferredBelief, + Program, + SemanticBelief, +) class TextBeliefExtractorAgent(BaseAgent): @@ -12,46 +27,110 @@ class TextBeliefExtractorAgent(BaseAgent): This agent is responsible for processing raw text (e.g., from speech transcription) and extracting semantic beliefs from it. - In the current demonstration version, it performs a simple wrapping of the user's input - into a ``user_said`` belief. In a full implementation, this agent would likely interact - with an LLM or NLU engine to extract intent, entities, and other structured information. + It uses the available beliefs received from the program manager to try to extract beliefs from a + user's message, sends and updated beliefs to the BDI core, and forms a ``user_said`` belief from + the message itself. """ + def __init__(self, name: str): + super().__init__(name) + self.beliefs = {} + self.available_beliefs = [] + self.conversation = ChatHistory(messages=[]) + async def setup(self): """ Initialize the agent and its resources. """ - self.logger.info("Settting up %s.", self.name) - # Setup LLM belief context if needed (currently demo is just passthrough) - self.beliefs = {"mood": ["X"], "car": ["Y"]} + self.logger.info("Setting up %s.", self.name) async def handle_message(self, msg: InternalMessage): """ - Handle incoming messages, primarily from the Transcription Agent. + Handle incoming messages. Expect messages from the Transcriber agent, LLM agent, and the + Program manager agent. - :param msg: The received message containing transcribed text. + :param msg: The received message. """ sender = msg.sender - if sender == settings.agent_settings.transcription_name: - self.logger.debug("Received text from transcriber: %s", msg.body) - await self._process_transcription_demo(msg.body) - else: - self.logger.info("Discarding message from %s", sender) - async def _process_transcription_demo(self, txt: str): + match sender: + case settings.agent_settings.transcription_name: + self.logger.debug("Received text from transcriber: %s", msg.body) + self._apply_conversation_message(ChatMessage(role="user", content=msg.body)) + await self._infer_new_beliefs() + await self._user_said(msg.body) + case settings.agent_settings.llm_name: + self.logger.debug("Received text from LLM: %s", msg.body) + self._apply_conversation_message(ChatMessage(role="assistant", content=msg.body)) + case settings.agent_settings.bdi_program_manager_name: + self._handle_program_manager_message(msg) + case _: + self.logger.info("Discarding message from %s", sender) + return + + def _apply_conversation_message(self, message: ChatMessage): """ - Process the transcribed text and generate beliefs. + Save the chat message to our conversation history, taking into account the conversation + length limit. - **Demo Implementation:** - Currently, this method takes the raw text ``txt`` and wraps it into a belief structure: - ``user_said("txt")``. - - This belief is then sent to the :class:`BDIBeliefCollectorAgent`. - - :param txt: The raw transcribed text string. + :param message: The chat message to add to the conversation history. """ - # For demo, just wrapping user text as user_said belief - belief = {"beliefs": {"user_said": [txt]}, "type": "belief_extraction_text"} + length_limit = settings.behaviour_settings.conversation_history_length_limit + self.conversation.messages = (self.conversation.messages + [message])[-length_limit:] + + def _handle_program_manager_message(self, msg: InternalMessage): + """ + Handle a message from the program manager: extract available beliefs from it. + + :param msg: The received message from the program manager. + """ + try: + program = Program.model_validate_json(msg.body) + except ValidationError: + self.logger.warning( + "Received message from program manager but it is not a valid program." + ) + return + + self.logger.debug("Received a program from the program manager.") + + self.available_beliefs = self._extract_basic_beliefs_from_program(program) + + # TODO Copied from an incomplete version of the program manager. Use that one instead. + @staticmethod + def _extract_basic_beliefs_from_program(program: Program) -> list[SemanticBelief]: + beliefs = [] + + for phase in program.phases: + for norm in phase.norms: + if isinstance(norm, ConditionalNorm): + beliefs += TextBeliefExtractorAgent._extract_basic_beliefs_from_belief( + norm.condition + ) + + for trigger in phase.triggers: + beliefs += TextBeliefExtractorAgent._extract_basic_beliefs_from_belief( + trigger.condition + ) + + return beliefs + + # TODO Copied from an incomplete version of the program manager. Use that one instead. + @staticmethod + def _extract_basic_beliefs_from_belief(belief: Belief) -> list[SemanticBelief]: + if isinstance(belief, InferredBelief): + return TextBeliefExtractorAgent._extract_basic_beliefs_from_belief( + belief.left + ) + TextBeliefExtractorAgent._extract_basic_beliefs_from_belief(belief.right) + return [belief] + + async def _user_said(self, text: str): + """ + Create a belief for the user's full speech. + + :param text: User's transcribed text. + """ + belief = {"beliefs": {"user_said": [text]}, "type": "belief_extraction_text"} payload = json.dumps(belief) belief_msg = InternalMessage( @@ -60,6 +139,200 @@ class TextBeliefExtractorAgent(BaseAgent): body=payload, thread="beliefs", ) - await self.send(belief_msg) - self.logger.info("Sent %d beliefs to the belief collector.", len(belief["beliefs"])) + + async def _infer_new_beliefs(self): + """ + Process conversation history to extract beliefs, semantically. Any changed beliefs are sent + to the BDI core. + """ + # Return instantly if there are no beliefs to infer + if not self.available_beliefs: + return + + candidate_beliefs = await self._infer_turn() + new_beliefs: list[InternalBelief] = [] + for belief_key, belief_value in candidate_beliefs.items(): + if belief_value is None: + continue + old_belief_value = self.beliefs.get(belief_key) + # TODO: Do we need this check? Can we send the same beliefs multiple times? + if belief_value == old_belief_value: + continue + self.beliefs[belief_key] = belief_value + new_beliefs.append( + InternalBelief(name=belief_key, arguments=[belief_value], replace=True), + ) + + beliefs_message = InternalMessage( + to=settings.agent_settings.bdi_core_name, + sender=self.name, + body=BeliefMessage(beliefs=new_beliefs).model_dump_json(), + thread="beliefs", + ) + await self.send(beliefs_message) + + @staticmethod + def _split_into_chunks[T](items: list[T], n: int) -> list[list[T]]: + k, m = divmod(len(items), n) + return [items[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] + + async def _infer_turn(self) -> dict: + """ + Process the stored conversation history to extract semantic beliefs. Returns a list of + beliefs that have been set to ``True``, ``False`` or ``None``. + + :return: A dict mapping belief names to a value ``True``, ``False`` or ``None``. + """ + n_parallel = min(settings.llm_settings.n_parallel - 1, len(self.available_beliefs)) + all_beliefs = await asyncio.gather( + *[ + self._infer_beliefs(self.conversation, beliefs) + for beliefs in self._split_into_chunks(self.available_beliefs, n_parallel) + ] + ) + retval = {} + for beliefs in all_beliefs: + if beliefs is None: + continue + retval.update(beliefs) + return retval + + @staticmethod + def _create_belief_schema(belief: SemanticBelief) -> tuple[str, dict]: + # TODO: use real belief names + return belief.name or slugify(belief.description), { + "type": ["boolean", "null"], + "description": belief.description, + } + + @staticmethod + def _create_beliefs_schema(beliefs: list[SemanticBelief]) -> dict: + belief_schemas = [ + TextBeliefExtractorAgent._create_belief_schema(belief) for belief in beliefs + ] + + return { + "type": "object", + "properties": dict(belief_schemas), + "required": [name for name, _ in belief_schemas], + } + + @staticmethod + def _format_message(message: ChatMessage): + return f"{message.role.upper()}:\n{message.content}" + + @staticmethod + def _format_conversation(conversation: ChatHistory): + return "\n\n".join( + [TextBeliefExtractorAgent._format_message(message) for message in conversation.messages] + ) + + @staticmethod + def _format_beliefs(beliefs: list[SemanticBelief]): + # TODO: use real belief names + return "\n".join( + [ + f"- {belief.name or slugify(belief.description)}: {belief.description}" + for belief in beliefs + ] + ) + + async def _infer_beliefs( + self, + conversation: ChatHistory, + beliefs: list[SemanticBelief], + ) -> dict | None: + """ + Infer given beliefs based on the given conversation. + :param conversation: The conversation to infer beliefs from. + :param beliefs: The beliefs to infer. + :return: A dict containing belief names and a boolean whether they hold, or None if the + belief cannot be inferred based on the given conversation. + """ + example = { + "example_belief": True, + } + + prompt = f"""{self._format_conversation(conversation)} + +Given the above conversation, what beliefs can be inferred? +If there is no relevant information about a belief belief, give null. +In case messages conflict, prefer using the most recent messages for inference. + +Choose from the following list of beliefs, formatted as (belief_name, description): +{self._format_beliefs(beliefs)} + +Respond with a JSON similar to the following, but with the property names as given above: +{json.dumps(example, indent=2)} +""" + + schema = self._create_beliefs_schema(beliefs) + + return await self._retry_query_llm(prompt, schema) + + async def _retry_query_llm(self, prompt: str, schema: dict, tries: int = 3) -> dict | None: + """ + Query the LLM with the given prompt and schema, return an instance of a dict conforming + to this schema. Try ``tries`` times, or return None. + + :param prompt: Prompt to be queried. + :param schema: Schema to be queried. + :return: An instance of a dict conforming to this schema, or None if failed. + """ + try_count = 0 + while try_count < tries: + try_count += 1 + + try: + return await self._query_llm(prompt, schema) + except (httpx.HTTPStatusError, json.JSONDecodeError, KeyError) as e: + if try_count < tries: + continue + self.logger.exception( + "Failed to get LLM response after %d tries.", + try_count, + exc_info=e, + ) + + return None + + @staticmethod + async def _query_llm(prompt: str, schema: dict) -> dict: + """ + Query an LLM with the given prompt and schema, return an instance of a dict conforming to + that schema. + + :param prompt: The prompt to be queried. + :param schema: Schema to use during response. + :return: A dict conforming to this schema. + :raises httpx.HTTPStatusError: If the LLM server responded with an error. + :raises json.JSONDecodeError: If the LLM response was not valid JSON. May happen if the + response was cut off early due to length limitations. + :raises KeyError: If the LLM server responded with no error, but the response was invalid. + """ + async with httpx.AsyncClient() as client: + response = await client.post( + settings.llm_settings.local_llm_url, + json={ + "model": settings.llm_settings.local_llm_model, + "messages": [{"role": "user", "content": prompt}], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "Beliefs", + "strict": True, + "schema": schema, + }, + }, + "reasoning_effort": "low", + "temperature": settings.llm_settings.code_temperature, + "stream": False, + }, + timeout=None, + ) + response.raise_for_status() + + response_json = response.json() + json_message = response_json["choices"][0]["message"]["content"] + return json.loads(json_message) diff --git a/src/control_backend/agents/llm/llm_agent.py b/src/control_backend/agents/llm/llm_agent.py index 55099e2..17edec9 100644 --- a/src/control_backend/agents/llm/llm_agent.py +++ b/src/control_backend/agents/llm/llm_agent.py @@ -64,11 +64,12 @@ class LLMAgent(BaseAgent): :param message: The parsed prompt message containing text, norms, and goals. """ + full_message = "" async for chunk in self._query_llm(message.text, message.norms, message.goals): await self._send_reply(chunk) - self.logger.debug( - "Finished processing BDI message. Response sent in chunks to BDI core." - ) + full_message += chunk + self.logger.debug("Finished processing BDI message. Response sent in chunks to BDI core.") + await self._send_full_reply(full_message) async def _send_reply(self, msg: str): """ @@ -83,6 +84,19 @@ class LLMAgent(BaseAgent): ) await self.send(reply) + async def _send_full_reply(self, msg: str): + """ + Sends a response message (full) to agents that need it. + + :param msg: The text content of the message. + """ + message = InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=self.name, + body=msg, + ) + await self.send(message) + async def _query_llm( self, prompt: str, norms: list[str], goals: list[str] ) -> AsyncGenerator[str]: @@ -172,7 +186,7 @@ class LLMAgent(BaseAgent): json={ "model": settings.llm_settings.local_llm_model, "messages": messages, - "temperature": 0.3, + "temperature": settings.llm_settings.chat_temperature, "stream": True, }, ) as response: diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index 927985b..1a2560a 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -65,6 +65,7 @@ class BehaviourSettings(BaseModel): :ivar transcription_words_per_minute: Estimated words per minute for transcription timing. :ivar transcription_words_per_token: Estimated words per token for transcription timing. :ivar transcription_token_buffer: Buffer for transcription tokens. + :ivar conversation_history_length_limit: The maximum amount of messages to extract beliefs from. """ sleep_s: float = 1.0 @@ -82,6 +83,9 @@ class BehaviourSettings(BaseModel): transcription_words_per_token: float = 0.75 # (3 words = 4 tokens) transcription_token_buffer: int = 10 + # Text belief extractor settings + conversation_history_length_limit = 10 + class LLMSettings(BaseModel): """ @@ -89,10 +93,17 @@ class LLMSettings(BaseModel): :ivar local_llm_url: URL for the local LLM API. :ivar local_llm_model: Name of the local LLM model to use. + :ivar chat_temperature: The temperature to use while generating chat responses. + :ivar code_temperature: The temperature to use while generating code-like responses like during + belief inference. + :ivar n_parallel: The number of parallel calls allowed to be made to the LLM. """ local_llm_url: str = "http://localhost:1234/v1/chat/completions" local_llm_model: str = "gpt-oss" + chat_temperature = 1.0 + code_temperature = 0.3 + n_parallel: int = 4 class VADSettings(BaseModel): diff --git a/src/control_backend/schemas/chat_history.py b/src/control_backend/schemas/chat_history.py new file mode 100644 index 0000000..52fc224 --- /dev/null +++ b/src/control_backend/schemas/chat_history.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel + + +class ChatMessage(BaseModel): + role: str + content: str + + +class ChatHistory(BaseModel): + messages: list[ChatMessage] diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 28969b9..529a23d 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -1,64 +1,201 @@ -from pydantic import BaseModel +from enum import Enum +from typing import Literal + +from pydantic import UUID4, BaseModel -class Norm(BaseModel): +class ProgramElement(BaseModel): """ - Represents a behavioral norm. + Represents a basic element of our behavior program. + :ivar name: The researcher-assigned name of the element. :ivar id: Unique identifier. - :ivar label: Human-readable label. - :ivar norm: The actual norm text describing the behavior. """ - id: str - label: str - norm: str + name: str + id: UUID4 -class Goal(BaseModel): +class LogicalOperator(Enum): + AND = "AND" + OR = "OR" + + +type Belief = KeywordBelief | SemanticBelief | InferredBelief +type BasicBelief = KeywordBelief | SemanticBelief + + +class KeywordBelief(ProgramElement): """ - Represents an objective to be achieved. + Represents a belief that is set when the user spoken text contains a certain keyword. - :ivar id: Unique identifier. - :ivar label: Human-readable label. - :ivar description: Detailed description of the goal. - :ivar achieved: Status flag indicating if the goal has been met. + :ivar keyword: The keyword on which this belief gets set. """ - id: str - label: str - description: str - achieved: bool - - -class TriggerKeyword(BaseModel): - id: str + name: str = "" keyword: str -class KeywordTrigger(BaseModel): - id: str - label: str - type: str - keywords: list[TriggerKeyword] +class SemanticBelief(ProgramElement): + """ + Represents a belief that is set by semantic LLM validation. + + :ivar description: Description of how to form the belief, used by the LLM. + """ + + name: str = "" + description: str -class Phase(BaseModel): +class InferredBelief(ProgramElement): + """ + Represents a belief that gets formed by combining two beliefs with a logical AND or OR. + + These beliefs can also be :class:`InferredBelief`, leading to arbitrarily deep nesting. + + :ivar operator: The logical operator to apply. + :ivar left: The left part of the logical expression. + :ivar right: The right part of the logical expression. + """ + + name: str = "" + operator: LogicalOperator + left: Belief + right: Belief + + +type Norm = BasicNorm | ConditionalNorm + + +class BasicNorm(ProgramElement): + """ + Represents a behavioral norm. + + :ivar norm: The actual norm text describing the behavior. + :ivar critical: When true, this norm should absolutely not be violated (checked separately). + """ + + name: str = "" + norm: str + critical: bool = False + + +class ConditionalNorm(BasicNorm): + """ + Represents a norm that is only active when a condition is met (i.e., a certain belief holds). + + :ivar condition: When to activate this norm. + """ + + condition: Belief + + +type PlanElement = Goal | Action + + +class Plan(ProgramElement): + """ + Represents a list of steps to execute. Each of these steps can be a goal (with its own plan) + or a simple action. + + :ivar steps: The actions or subgoals to execute, in order. + """ + + name: str = "" + steps: list[PlanElement] + + +class Goal(ProgramElement): + """ + Represents an objective to be achieved. To reach the goal, we should execute + the corresponding plan. If we can fail to achieve a goal after executing the plan, + for example when the achieving of the goal is dependent on the user's reply, this means + that the achieved status will be set from somewhere else in the program. + + :ivar plan: The plan to execute. + :ivar can_fail: Whether we can fail to achieve the goal after executing the plan. + """ + + plan: Plan + can_fail: bool = True + + +type Action = SpeechAction | GestureAction | LLMAction + + +class SpeechAction(ProgramElement): + """ + Represents the action of the robot speaking a literal text. + + :ivar text: The text to speak. + """ + + name: str = "" + text: str + + +class Gesture(BaseModel): + """ + Represents a gesture to be performed. Can be either a single gesture, + or a random gesture from a category (tag). + + :ivar type: The type of the gesture, "tag" or "single". + :ivar name: The name of the single gesture or tag. + """ + + type: Literal["tag", "single"] + name: str + + +class GestureAction(ProgramElement): + """ + Represents the action of the robot performing a gesture. + + :ivar gesture: The gesture to perform. + """ + + name: str = "" + gesture: Gesture + + +class LLMAction(ProgramElement): + """ + Represents the action of letting an LLM generate a reply based on its chat history + and an additional goal added in the prompt. + + :ivar goal: The extra (temporary) goal to add to the LLM. + """ + + name: str = "" + goal: str + + +class Trigger(ProgramElement): + """ + Represents a belief-based trigger. When a belief is set, the corresponding plan is executed. + + :ivar condition: When to activate the trigger. + :ivar plan: The plan to execute. + """ + + name: str = "" + condition: Belief + plan: Plan + + +class Phase(ProgramElement): """ A distinct phase within a program, containing norms, goals, and triggers. - :ivar id: Unique identifier. - :ivar label: Human-readable label. :ivar norms: List of norms active in this phase. :ivar goals: List of goals to pursue in this phase. :ivar triggers: List of triggers that define transitions out of this phase. """ - id: str - label: str + name: str = "" norms: list[Norm] goals: list[Goal] - triggers: list[KeywordTrigger] + triggers: list[Trigger] class Program(BaseModel): From 71cefdfef3c29aa98453fdb0dac22d466b5c095c Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:14:49 +0100 Subject: [PATCH 13/90] fix: add types to all config properties ref: N25B-380 --- src/control_backend/core/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index 1a2560a..8a7267c 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -84,7 +84,7 @@ class BehaviourSettings(BaseModel): transcription_token_buffer: int = 10 # Text belief extractor settings - conversation_history_length_limit = 10 + conversation_history_length_limit: int = 10 class LLMSettings(BaseModel): @@ -101,8 +101,8 @@ class LLMSettings(BaseModel): local_llm_url: str = "http://localhost:1234/v1/chat/completions" local_llm_model: str = "gpt-oss" - chat_temperature = 1.0 - code_temperature = 0.3 + chat_temperature: float = 1.0 + code_temperature: float = 0.3 n_parallel: int = 4 From 3253760ef195306e4131b03b01c596c0e9e97b20 Mon Sep 17 00:00:00 2001 From: Kasper Date: Tue, 23 Dec 2025 17:30:35 +0100 Subject: [PATCH 14/90] feat: new AST representation File names will be changed eventually. ref: N25B-376 --- src/control_backend/agents/bdi/astv2.py | 272 ++++++++++++++++++++++++ src/control_backend/agents/bdi/gen.py | 0 src/control_backend/agents/bdi/test.asl | 0 src/control_backend/schemas/program.py | 13 +- 4 files changed, 279 insertions(+), 6 deletions(-) create mode 100644 src/control_backend/agents/bdi/astv2.py create mode 100644 src/control_backend/agents/bdi/gen.py create mode 100644 src/control_backend/agents/bdi/test.asl diff --git a/src/control_backend/agents/bdi/astv2.py b/src/control_backend/agents/bdi/astv2.py new file mode 100644 index 0000000..f88fb6a --- /dev/null +++ b/src/control_backend/agents/bdi/astv2.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import StrEnum + + +class AstNode(ABC): + """ + Abstract base class for all elements of an AgentSpeak program. + """ + + @abstractmethod + def _to_agentspeak(self) -> str: + """ + Generates the AgentSpeak code string. + """ + pass + + def __str__(self) -> str: + return self._to_agentspeak() + + +class AstExpression(AstNode, ABC): + """ + Intermediate class for anything that can be used in a logical expression. + """ + + def __and__(self, other: ExprCoalescible) -> AstBinaryOp: + return AstBinaryOp(self, BinaryOperatorType.AND, _coalesce_expr(other)) + + def __or__(self, other: ExprCoalescible) -> AstBinaryOp: + return AstBinaryOp(self, BinaryOperatorType.OR, _coalesce_expr(other)) + + def __invert__(self) -> AstLogicalExpression: + if isinstance(self, AstLogicalExpression): + self.negated = not self.negated + return self + return AstLogicalExpression(self, negated=True) + + +type ExprCoalescible = AstExpression | str | int | float + + +def _coalesce_expr(value: ExprCoalescible) -> AstExpression: + if isinstance(value, AstExpression): + return value + if isinstance(value, str): + return AstString(value) + if isinstance(value, (int, float)): + return AstNumber(value) + raise TypeError(f"Cannot coalesce type {type(value)} into an AstTerm.") + + +@dataclass +class AstTerm(AstExpression, ABC): + """ + Base class for terms appearing inside literals. + """ + + def __ge__(self, other: ExprCoalescible) -> AstBinaryOp: + return AstBinaryOp(self, BinaryOperatorType.GREATER_EQUALS, _coalesce_expr(other)) + + def __gt__(self, other: ExprCoalescible) -> AstBinaryOp: + return AstBinaryOp(self, BinaryOperatorType.GREATER_THAN, _coalesce_expr(other)) + + def __le__(self, other: ExprCoalescible) -> AstBinaryOp: + return AstBinaryOp(self, BinaryOperatorType.LESS_EQUALS, _coalesce_expr(other)) + + def __lt__(self, other: ExprCoalescible) -> AstBinaryOp: + return AstBinaryOp(self, BinaryOperatorType.LESS_THAN, _coalesce_expr(other)) + + def __eq__(self, other: ExprCoalescible) -> AstBinaryOp: + return AstBinaryOp(self, BinaryOperatorType.EQUALS, _coalesce_expr(other)) + + def __ne__(self, other: ExprCoalescible) -> AstBinaryOp: + return AstBinaryOp(self, BinaryOperatorType.NOT_EQUALS, _coalesce_expr(other)) + + +@dataclass +class AstAtom(AstTerm): + """ + Grounded expression in all lowercase. + """ + + value: str + + def _to_agentspeak(self) -> str: + return self.value.lower() + + +@dataclass +class AstVar(AstTerm): + """ + Ungrounded variable expression. First letter capitalized. + """ + + name: str + + def _to_agentspeak(self) -> str: + return self.name.capitalize() + + +@dataclass +class AstNumber(AstTerm): + value: int | float + + def _to_agentspeak(self) -> str: + return str(self.value) + + +@dataclass +class AstString(AstTerm): + value: str + + def _to_agentspeak(self) -> str: + return f'"{self.value}"' + + +@dataclass +class AstLiteral(AstTerm): + functor: str + terms: list[AstTerm] = field(default_factory=list) + + def _to_agentspeak(self) -> str: + if not self.terms: + return self.functor + args = ", ".join(map(str, self.terms)) + return f"{self.functor}({args})" + + +class BinaryOperatorType(StrEnum): + AND = "&" + OR = "|" + GREATER_THAN = ">" + LESS_THAN = "<" + EQUALS = "==" + NOT_EQUALS = "\\==" + GREATER_EQUALS = ">=" + LESS_EQUALS = "<=" + + +@dataclass +class AstBinaryOp(AstExpression): + left: AstExpression + operator: BinaryOperatorType + right: AstExpression + + def __post_init__(self): + self.left = _as_logical(self.left) + self.right = _as_logical(self.right) + + def _to_agentspeak(self) -> str: + l_str = str(self.left) + r_str = str(self.right) + + assert isinstance(self.left, AstLogicalExpression) + assert isinstance(self.right, AstLogicalExpression) + + if isinstance(self.left.expression, AstBinaryOp) or self.left.negated: + l_str = f"({l_str})" + if isinstance(self.right.expression, AstBinaryOp) or self.right.negated: + r_str = f"({r_str})" + + return f"{l_str} {self.operator.value} {r_str}" + + +@dataclass +class AstLogicalExpression(AstExpression): + expression: AstExpression + negated: bool = False + + def _to_agentspeak(self) -> str: + expr_str = str(self.expression) + if isinstance(self.expression, AstBinaryOp) and self.negated: + expr_str = f"({expr_str})" + return f"{'not ' if self.negated else ''}{expr_str}" + + +def _as_logical(expr: AstExpression) -> AstLogicalExpression: + if isinstance(expr, AstLogicalExpression): + return expr + return AstLogicalExpression(expr) + + +class StatementType(StrEnum): + EMPTY = "" + DO_ACTION = "." + ACHIEVE_GOAL = "!" + # TEST_GOAL = "?" # TODO + ADD_BELIEF = "+" + REMOVE_BELIEF = "-" + + +@dataclass +class AstStatement(AstNode): + """ + A statement that can appear inside a plan. + """ + + type: StatementType + expression: AstExpression + + def _to_agentspeak(self) -> str: + return f"{self.type.value}{self.expression}" + + +@dataclass +class AstRule(AstNode): + result: AstExpression + condition: AstExpression | None = None + + def __post_init__(self): + if self.condition is not None: + self.condition = _as_logical(self.condition) + + def _to_agentspeak(self) -> str: + if not self.condition: + return f"{self.result}." + return f"{self.result} :- {self.condition}." + + +class TriggerType(StrEnum): + ADDED_BELIEF = "+" + # REMOVED_BELIEF = "-" # TODO + # MODIFIED_BELIEF = "^" # TODO + ADDED_GOAL = "+!" + # REMOVED_GOAL = "-!" # TODO + + +@dataclass +class AstPlan(AstNode): + type: TriggerType + trigger_literal: AstExpression + context: list[AstExpression] + body: list[AstStatement] + + def _to_agentspeak(self) -> str: + assert isinstance(self.trigger_literal, AstLiteral) + + indent = " " * 6 + colon = " : " + arrow = " <- " + + lines = [] + + lines.append(f"{self.type.value}{self.trigger_literal}") + + if self.context: + lines.append(colon + f" &\n{indent}".join(str(c) for c in self.context)) + + if self.body: + lines.append(arrow + f";\n{indent}".join(str(s) for s in self.body) + ".") + + lines.append("") + + return "\n".join(lines) + + +@dataclass +class AstProgram(AstNode): + rules: list[AstRule] = field(default_factory=list) + plans: list[AstPlan] = field(default_factory=list) + + def _to_agentspeak(self) -> str: + lines = [] + lines.extend(map(str, self.rules)) + + lines.extend(["", ""]) + lines.extend(map(str, self.plans)) + + return "\n".join(lines) diff --git a/src/control_backend/agents/bdi/gen.py b/src/control_backend/agents/bdi/gen.py new file mode 100644 index 0000000..e69de29 diff --git a/src/control_backend/agents/bdi/test.asl b/src/control_backend/agents/bdi/test.asl new file mode 100644 index 0000000..e69de29 diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 529a23d..5a8caa9 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -64,10 +64,13 @@ class InferredBelief(ProgramElement): right: Belief -type Norm = BasicNorm | ConditionalNorm +class Norm(ProgramElement): + name: str = "" + norm: str + critical: bool = False -class BasicNorm(ProgramElement): +class BasicNorm(Norm): """ Represents a behavioral norm. @@ -75,12 +78,10 @@ class BasicNorm(ProgramElement): :ivar critical: When true, this norm should absolutely not be violated (checked separately). """ - name: str = "" - norm: str - critical: bool = False + pass -class ConditionalNorm(BasicNorm): +class ConditionalNorm(Norm): """ Represents a norm that is only active when a condition is met (i.e., a certain belief holds). From 57b1276cb5f569dd5a6a17f9ade8f1035922ce7d Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Mon, 29 Dec 2025 12:31:51 +0100 Subject: [PATCH 15/90] test: make tests work again after changing Program schema ref: N25B-380 --- .../agents/bdi/bdi_core_agent.py | 6 +- .../agents/bdi/test_bdi_program_manager.py | 54 ++++++++++------ test/unit/agents/bdi/test_text_extractor.py | 11 +--- test/unit/agents/llm/test_llm_agent.py | 2 +- .../api/v1/endpoints/test_program_endpoint.py | 56 ++++++++++------- test/unit/schemas/test_ui_program_message.py | 62 +++++++++++-------- 6 files changed, 110 insertions(+), 81 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 8ff271c..427e024 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -89,9 +89,9 @@ class BDICoreAgent(BaseAgent): the agent has deferred intentions (deadlines). """ while self._running: - # await ( - # self._wake_bdi_loop.wait() - # ) # gets set whenever there's an update to the belief base + await ( + self._wake_bdi_loop.wait() + ) # gets set whenever there's an update to the belief base # Agent knows when it's expected to have to do its next thing maybe_more_work = True diff --git a/test/unit/agents/bdi/test_bdi_program_manager.py b/test/unit/agents/bdi/test_bdi_program_manager.py index a54360c..d16bc43 100644 --- a/test/unit/agents/bdi/test_bdi_program_manager.py +++ b/test/unit/agents/bdi/test_bdi_program_manager.py @@ -1,6 +1,6 @@ import asyncio -import json import sys +import uuid from unittest.mock import AsyncMock import pytest @@ -8,31 +8,45 @@ import pytest from control_backend.agents.bdi.bdi_program_manager import BDIProgramManager from control_backend.core.agent_system import InternalMessage from control_backend.schemas.belief_message import BeliefMessage -from control_backend.schemas.program import Program +from control_backend.schemas.program import BasicNorm, Goal, Phase, Plan, Program # Fix Windows Proactor loop for zmq if sys.platform.startswith("win"): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -def make_valid_program_json(norm="N1", goal="G1"): - return json.dumps( - { - "phases": [ - { - "id": "phase1", - "label": "Phase 1", - "triggers": [], - "norms": [{"id": "n1", "label": "Norm 1", "norm": norm}], - "goals": [ - {"id": "g1", "label": "Goal 1", "description": goal, "achieved": False} - ], - } - ] - } - ) +def make_valid_program_json(norm="N1", goal="G1") -> str: + return Program( + phases=[ + Phase( + id=uuid.uuid4(), + name="Basic Phase", + norms=[ + BasicNorm( + id=uuid.uuid4(), + name=norm, + norm=norm, + ), + ], + goals=[ + Goal( + id=uuid.uuid4(), + name=goal, + plan=Plan( + id=uuid.uuid4(), + name="Goal Plan", + steps=[], + ), + can_fail=False, + ), + ], + triggers=[], + ), + ], + ).model_dump_json() +@pytest.mark.skip(reason="Functionality being rebuilt.") @pytest.mark.asyncio async def test_send_to_bdi(): manager = BDIProgramManager(name="program_manager_test") @@ -73,5 +87,5 @@ async def test_receive_programs_valid_and_invalid(): # Only valid Program should have triggered _send_to_bdi assert manager._send_to_bdi.await_count == 1 forwarded: Program = manager._send_to_bdi.await_args[0][0] - assert forwarded.phases[0].norms[0].norm == "N1" - assert forwarded.phases[0].goals[0].description == "G1" + assert forwarded.phases[0].norms[0].name == "N1" + assert forwarded.phases[0].goals[0].name == "G1" diff --git a/test/unit/agents/bdi/test_text_extractor.py b/test/unit/agents/bdi/test_text_extractor.py index 895fef0..c51571a 100644 --- a/test/unit/agents/bdi/test_text_extractor.py +++ b/test/unit/agents/bdi/test_text_extractor.py @@ -45,10 +45,10 @@ async def test_handle_message_from_transcriber(agent, mock_settings): @pytest.mark.asyncio -async def test_process_transcription_demo(agent, mock_settings): +async def test_process_user_said(agent, mock_settings): transcription = "this is a test" - await agent._process_transcription_demo(transcription) + await agent._user_said(transcription) agent.send.assert_awaited_once() # noqa # `agent.send` has no such property, but we mock it. sent: InternalMessage = agent.send.call_args.args[0] # noqa @@ -56,10 +56,3 @@ async def test_process_transcription_demo(agent, mock_settings): assert sent.thread == "beliefs" parsed = json.loads(sent.body) assert parsed["beliefs"]["user_said"] == [transcription] - - -@pytest.mark.asyncio -async def test_setup_initializes_beliefs(agent): - """Covers the setup method and ensures beliefs are initialized.""" - await agent.setup() - assert agent.beliefs == {"mood": ["X"], "car": ["Y"]} diff --git a/test/unit/agents/llm/test_llm_agent.py b/test/unit/agents/llm/test_llm_agent.py index 5e84d8d..6cda4da 100644 --- a/test/unit/agents/llm/test_llm_agent.py +++ b/test/unit/agents/llm/test_llm_agent.py @@ -66,7 +66,7 @@ async def test_llm_processing_success(mock_httpx_client, mock_settings): # "Hello world." constitutes one sentence/chunk based on punctuation split # The agent should call send once with the full sentence assert agent.send.called - args = agent.send.call_args[0][0] + args = agent.send.call_args_list[0][0][0] assert args.to == mock_settings.agent_settings.bdi_core_name assert "Hello world." in args.body diff --git a/test/unit/api/v1/endpoints/test_program_endpoint.py b/test/unit/api/v1/endpoints/test_program_endpoint.py index 178159c..379767a 100644 --- a/test/unit/api/v1/endpoints/test_program_endpoint.py +++ b/test/unit/api/v1/endpoints/test_program_endpoint.py @@ -1,4 +1,5 @@ import json +import uuid from unittest.mock import AsyncMock import pytest @@ -6,7 +7,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from control_backend.api.v1.endpoints import program -from control_backend.schemas.program import Program +from control_backend.schemas.program import BasicNorm, Goal, Phase, Plan, Program @pytest.fixture @@ -25,29 +26,37 @@ def client(app): def make_valid_program_dict(): """Helper to create a valid Program JSON structure.""" - return { - "phases": [ - { - "id": "phase1", - "label": "basephase", - "norms": [{"id": "n1", "label": "norm", "norm": "be nice"}], - "goals": [ - {"id": "g1", "label": "goal", "description": "test goal", "achieved": False} + # Converting to JSON using Pydantic because it knows how to convert a UUID object + program_json_str = Program( + phases=[ + Phase( + id=uuid.uuid4(), + name="Basic Phase", + norms=[ + BasicNorm( + id=uuid.uuid4(), + name="Some norm", + norm="Do normal.", + ), ], - "triggers": [ - { - "id": "t1", - "label": "trigger", - "type": "keywords", - "keywords": [ - {"id": "kw1", "keyword": "keyword1"}, - {"id": "kw2", "keyword": "keyword2"}, - ], - }, + goals=[ + Goal( + id=uuid.uuid4(), + name="Some goal", + plan=Plan( + id=uuid.uuid4(), + name="Goal Plan", + steps=[], + ), + can_fail=False, + ), ], - } - ] - } + triggers=[], + ), + ], + ).model_dump_json() + # Converting back to a dict because that's what's expected + return json.loads(program_json_str) def test_receive_program_success(client): @@ -71,7 +80,8 @@ def test_receive_program_success(client): sent_bytes = args[0][1] sent_obj = json.loads(sent_bytes.decode()) - expected_obj = Program.model_validate(program_dict).model_dump() + # Converting to JSON using Pydantic because it knows how to handle UUIDs + expected_obj = json.loads(Program.model_validate(program_dict).model_dump_json()) assert sent_obj == expected_obj diff --git a/test/unit/schemas/test_ui_program_message.py b/test/unit/schemas/test_ui_program_message.py index 7ed544e..a9f96dd 100644 --- a/test/unit/schemas/test_ui_program_message.py +++ b/test/unit/schemas/test_ui_program_message.py @@ -1,49 +1,61 @@ +import uuid + import pytest from pydantic import ValidationError from control_backend.schemas.program import ( + BasicNorm, Goal, - KeywordTrigger, - Norm, + KeywordBelief, Phase, + Plan, Program, - TriggerKeyword, + Trigger, ) -def base_norm() -> Norm: - return Norm( - id="norm1", - label="testNorm", +def base_norm() -> BasicNorm: + return BasicNorm( + id=uuid.uuid4(), + name="testNormName", norm="testNormNorm", + critical=False, ) def base_goal() -> Goal: return Goal( - id="goal1", - label="testGoal", - description="testGoalDescription", - achieved=False, + id=uuid.uuid4(), + name="testGoalName", + plan=Plan( + id=uuid.uuid4(), + name="testGoalPlanName", + steps=[], + ), + can_fail=False, ) -def base_trigger() -> KeywordTrigger: - return KeywordTrigger( - id="trigger1", - label="testTrigger", - type="keywords", - keywords=[ - TriggerKeyword(id="keyword1", keyword="testKeyword1"), - TriggerKeyword(id="keyword1", keyword="testKeyword2"), - ], +def base_trigger() -> Trigger: + return Trigger( + id=uuid.uuid4(), + name="testTriggerName", + condition=KeywordBelief( + id=uuid.uuid4(), + name="testTriggerKeywordBeliefTriggerName", + keyword="Keyword", + ), + plan=Plan( + id=uuid.uuid4(), + name="testTriggerPlanName", + steps=[], + ), ) def base_phase() -> Phase: return Phase( - id="phase1", - label="basephase", + id=uuid.uuid4(), norms=[base_norm()], goals=[base_goal()], triggers=[base_trigger()], @@ -58,7 +70,7 @@ def invalid_program() -> dict: # wrong types inside phases list (not Phase objects) return { "phases": [ - {"id": "phase1"}, # incomplete + {"id": uuid.uuid4()}, # incomplete {"not_a_phase": True}, ] } @@ -77,8 +89,8 @@ def test_valid_deepprogram(): # validate nested components directly phase = validated.phases[0] assert isinstance(phase.goals[0], Goal) - assert isinstance(phase.triggers[0], KeywordTrigger) - assert isinstance(phase.norms[0], Norm) + assert isinstance(phase.triggers[0], Trigger) + assert isinstance(phase.norms[0], BasicNorm) def test_invalid_program(): From 42ee5c76d8401898308b384d3f48b6f1b74216ba Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Mon, 29 Dec 2025 17:12:02 +0100 Subject: [PATCH 16/90] test: create tests for belief extractor agent Includes changes in schemas. Change type of `norms` in `Program` imperceptibly, big changes in schema of `BeliefMessage` to support deleting beliefs. ref: N25B-380 --- .../agents/bdi/bdi_core_agent.py | 27 +- .../agents/bdi/belief_collector_agent.py | 2 +- .../agents/bdi/text_belief_extractor_agent.py | 27 +- src/control_backend/schemas/belief_message.py | 21 +- src/control_backend/schemas/program.py | 2 +- test/unit/agents/bdi/test_bdi_core_agent.py | 32 +- test/unit/agents/bdi/test_belief_collector.py | 2 +- .../agents/bdi/test_text_belief_extractor.py | 346 ++++++++++++++++++ test/unit/agents/bdi/test_text_extractor.py | 58 --- test/unit/schemas/test_ui_program_message.py | 105 ++++++ 10 files changed, 530 insertions(+), 92 deletions(-) create mode 100644 test/unit/agents/bdi/test_text_belief_extractor.py delete mode 100644 test/unit/agents/bdi/test_text_extractor.py diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 427e024..23c2808 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -11,7 +11,7 @@ from pydantic import ValidationError from control_backend.agents.base import BaseAgent from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings -from control_backend.schemas.belief_message import Belief, BeliefMessage +from control_backend.schemas.belief_message import BeliefMessage from control_backend.schemas.llm_prompt_message import LLMPromptMessage from control_backend.schemas.ri_message import SpeechCommand @@ -124,8 +124,8 @@ class BDICoreAgent(BaseAgent): if msg.thread == "beliefs": try: - beliefs = BeliefMessage.model_validate_json(msg.body).beliefs - self._apply_beliefs(beliefs) + belief_changes = BeliefMessage.model_validate_json(msg.body) + self._apply_belief_changes(belief_changes) except ValidationError: self.logger.exception("Error processing belief.") return @@ -145,21 +145,28 @@ class BDICoreAgent(BaseAgent): ) await self.send(out_msg) - def _apply_beliefs(self, beliefs: list[Belief]): + def _apply_belief_changes(self, belief_changes: BeliefMessage): """ Update the belief base with a list of new beliefs. - If ``replace=True`` is set on a belief, it removes all existing beliefs with that name - before adding the new one. + For beliefs in ``belief_changes.replace``, it removes all existing beliefs with that name + before adding one new one. + + :param belief_changes: The changes in beliefs to apply. """ - if not beliefs: + if not belief_changes.create and not belief_changes.replace and not belief_changes.delete: return - for belief in beliefs: - if belief.replace: - self._remove_all_with_name(belief.name) + for belief in belief_changes.create: self._add_belief(belief.name, belief.arguments) + for belief in belief_changes.replace: + self._remove_all_with_name(belief.name) + self._add_belief(belief.name, belief.arguments) + + for belief in belief_changes.delete: + self._remove_belief(belief.name, belief.arguments) + def _add_belief(self, name: str, args: list[str] = None): """ Add a single belief to the BDI agent. diff --git a/src/control_backend/agents/bdi/belief_collector_agent.py b/src/control_backend/agents/bdi/belief_collector_agent.py index 788cff1..ac0e2e5 100644 --- a/src/control_backend/agents/bdi/belief_collector_agent.py +++ b/src/control_backend/agents/bdi/belief_collector_agent.py @@ -144,7 +144,7 @@ class BDIBeliefCollectorAgent(BaseAgent): msg = InternalMessage( to=settings.agent_settings.bdi_core_name, sender=self.name, - body=BeliefMessage(beliefs=beliefs).model_dump_json(), + body=BeliefMessage(create=beliefs).model_dump_json(), thread="beliefs", ) diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index 0324573..5cc75d8 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -34,8 +34,8 @@ class TextBeliefExtractorAgent(BaseAgent): def __init__(self, name: str): super().__init__(name) - self.beliefs = {} - self.available_beliefs = [] + self.beliefs: dict[str, bool] = {} + self.available_beliefs: list[SemanticBelief] = [] self.conversation = ChatHistory(messages=[]) async def setup(self): @@ -151,23 +151,30 @@ class TextBeliefExtractorAgent(BaseAgent): return candidate_beliefs = await self._infer_turn() - new_beliefs: list[InternalBelief] = [] + belief_changes = BeliefMessage() for belief_key, belief_value in candidate_beliefs.items(): if belief_value is None: continue old_belief_value = self.beliefs.get(belief_key) - # TODO: Do we need this check? Can we send the same beliefs multiple times? if belief_value == old_belief_value: continue + self.beliefs[belief_key] = belief_value - new_beliefs.append( - InternalBelief(name=belief_key, arguments=[belief_value], replace=True), - ) + + belief = InternalBelief(name=belief_key, arguments=None) + if belief_value: + belief_changes.create.append(belief) + else: + belief_changes.delete.append(belief) + + # Return if there were no changes in beliefs + if not belief_changes.has_values(): + return beliefs_message = InternalMessage( to=settings.agent_settings.bdi_core_name, sender=self.name, - body=BeliefMessage(beliefs=new_beliefs).model_dump_json(), + body=belief_changes.model_dump_json(), thread="beliefs", ) await self.send(beliefs_message) @@ -184,7 +191,7 @@ class TextBeliefExtractorAgent(BaseAgent): :return: A dict mapping belief names to a value ``True``, ``False`` or ``None``. """ - n_parallel = min(settings.llm_settings.n_parallel - 1, len(self.available_beliefs)) + n_parallel = max(1, min(settings.llm_settings.n_parallel - 1, len(self.available_beliefs))) all_beliefs = await asyncio.gather( *[ self._infer_beliefs(self.conversation, beliefs) @@ -286,7 +293,7 @@ Respond with a JSON similar to the following, but with the property names as giv try: return await self._query_llm(prompt, schema) - except (httpx.HTTPStatusError, json.JSONDecodeError, KeyError) as e: + except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e: if try_count < tries: continue self.logger.exception( diff --git a/src/control_backend/schemas/belief_message.py b/src/control_backend/schemas/belief_message.py index deb1152..56a8a4a 100644 --- a/src/control_backend/schemas/belief_message.py +++ b/src/control_backend/schemas/belief_message.py @@ -6,18 +6,27 @@ class Belief(BaseModel): Represents a single belief in the BDI system. :ivar name: The functor or name of the belief (e.g., 'user_said'). - :ivar arguments: A list of string arguments for the belief. - :ivar replace: If True, existing beliefs with this name should be replaced by this one. + :ivar arguments: A list of string arguments for the belief, or None if the belief has no + arguments. """ name: str - arguments: list[str] - replace: bool = False + arguments: list[str] | None class BeliefMessage(BaseModel): """ - A container for transporting a list of beliefs between agents. + A container for communicating beliefs between agents. + + :ivar create: Beliefs to create. + :ivar delete: Beliefs to delete. + :ivar replace: Beliefs to replace. Deletes all beliefs with the same name, replacing them with + one new belief. """ - beliefs: list[Belief] + create: list[Belief] = [] + delete: list[Belief] = [] + replace: list[Belief] = [] + + def has_values(self) -> bool: + return len(self.create) > 0 or len(self.delete) > 0 or len(self.replace) > 0 diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 5a8caa9..be538b0 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -194,7 +194,7 @@ class Phase(ProgramElement): """ name: str = "" - norms: list[Norm] + norms: list[BasicNorm | ConditionalNorm] goals: list[Goal] triggers: list[Trigger] diff --git a/test/unit/agents/bdi/test_bdi_core_agent.py b/test/unit/agents/bdi/test_bdi_core_agent.py index 8d004fc..2325a57 100644 --- a/test/unit/agents/bdi/test_bdi_core_agent.py +++ b/test/unit/agents/bdi/test_bdi_core_agent.py @@ -51,7 +51,7 @@ async def test_handle_belief_collector_message(agent, mock_settings): msg = InternalMessage( to="bdi_agent", sender=mock_settings.agent_settings.bdi_belief_collector_name, - body=BeliefMessage(beliefs=beliefs).model_dump_json(), + body=BeliefMessage(create=beliefs).model_dump_json(), thread="beliefs", ) @@ -64,6 +64,26 @@ async def test_handle_belief_collector_message(agent, mock_settings): assert args[2] == agentspeak.Literal("user_said", (agentspeak.Literal("Hello"),)) +@pytest.mark.asyncio +async def test_handle_delete_belief_message(agent, mock_settings): + """Test that incoming beliefs to be deleted are removed from the BDI agent""" + beliefs = [Belief(name="user_said", arguments=["Hello"])] + + msg = InternalMessage( + to="bdi_agent", + sender=mock_settings.agent_settings.bdi_belief_collector_name, + body=BeliefMessage(delete=beliefs).model_dump_json(), + thread="beliefs", + ) + await agent.handle_message(msg) + + # Expect bdi_agent.call to be triggered to remove belief + args = agent.bdi_agent.call.call_args.args + assert args[0] == agentspeak.Trigger.removal + assert args[1] == agentspeak.GoalType.belief + assert args[2] == agentspeak.Literal("user_said", (agentspeak.Literal("Hello"),)) + + @pytest.mark.asyncio async def test_incorrect_belief_collector_message(agent, mock_settings): """Test that incorrect message format triggers an exception.""" @@ -128,7 +148,8 @@ def test_add_belief_sets_event(agent): agent._wake_bdi_loop = MagicMock() belief = Belief(name="test_belief", arguments=["a", "b"]) - agent._apply_beliefs([belief]) + belief_changes = BeliefMessage(replace=[belief]) + agent._apply_belief_changes(belief_changes) assert agent.bdi_agent.call.called agent._wake_bdi_loop.set.assert_called() @@ -137,7 +158,7 @@ def test_add_belief_sets_event(agent): def test_apply_beliefs_empty_returns(agent): """Line: if not beliefs: return""" agent._wake_bdi_loop = MagicMock() - agent._apply_beliefs([]) + agent._apply_belief_changes(BeliefMessage()) agent.bdi_agent.call.assert_not_called() agent._wake_bdi_loop.set.assert_not_called() @@ -220,8 +241,9 @@ def test_replace_belief_calls_remove_all(agent): agent._remove_all_with_name = MagicMock() agent._wake_bdi_loop = MagicMock() - belief = Belief(name="user_said", arguments=["Hello"], replace=True) - agent._apply_beliefs([belief]) + belief = Belief(name="user_said", arguments=["Hello"]) + belief_changes = BeliefMessage(replace=[belief]) + agent._apply_belief_changes(belief_changes) agent._remove_all_with_name.assert_called_with("user_said") diff --git a/test/unit/agents/bdi/test_belief_collector.py b/test/unit/agents/bdi/test_belief_collector.py index 67b2ed5..69db269 100644 --- a/test/unit/agents/bdi/test_belief_collector.py +++ b/test/unit/agents/bdi/test_belief_collector.py @@ -86,7 +86,7 @@ async def test_send_beliefs_to_bdi(agent): sent: InternalMessage = agent.send.call_args.args[0] assert sent.to == settings.agent_settings.bdi_core_name assert sent.thread == "beliefs" - assert json.loads(sent.body)["beliefs"] == [belief.model_dump() for belief in beliefs] + assert json.loads(sent.body)["create"] == [belief.model_dump() for belief in beliefs] @pytest.mark.asyncio diff --git a/test/unit/agents/bdi/test_text_belief_extractor.py b/test/unit/agents/bdi/test_text_belief_extractor.py new file mode 100644 index 0000000..827adbc --- /dev/null +++ b/test/unit/agents/bdi/test_text_belief_extractor.py @@ -0,0 +1,346 @@ +import json +import uuid +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from control_backend.agents.bdi import TextBeliefExtractorAgent +from control_backend.core.agent_system import InternalMessage +from control_backend.core.config import settings +from control_backend.schemas.belief_message import BeliefMessage +from control_backend.schemas.program import ( + ConditionalNorm, + LLMAction, + Phase, + Plan, + Program, + SemanticBelief, + Trigger, +) + + +@pytest.fixture +def agent(): + agent = TextBeliefExtractorAgent("text_belief_agent") + agent.send = AsyncMock() + agent._query_llm = AsyncMock() + return agent + + +@pytest.fixture +def sample_program(): + return Program( + phases=[ + Phase( + name="Some phase", + id=uuid.uuid4(), + norms=[ + ConditionalNorm( + name="Some norm", + id=uuid.uuid4(), + norm="Use nautical terms.", + critical=False, + condition=SemanticBelief( + name="is_pirate", + id=uuid.uuid4(), + description="The user is a pirate. Perhaps because they say " + "they are, or because they speak like a pirate " + 'with terms like "arr".', + ), + ), + ], + goals=[], + triggers=[ + Trigger( + name="Some trigger", + id=uuid.uuid4(), + condition=SemanticBelief( + name="no_more_booze", + id=uuid.uuid4(), + description="There is no more alcohol.", + ), + plan=Plan( + name="Some plan", + id=uuid.uuid4(), + steps=[ + LLMAction( + name="Some action", + id=uuid.uuid4(), + goal="Suggest eating chocolate instead.", + ), + ], + ), + ), + ], + ), + ], + ) + + +def make_msg(sender: str, body: str, thread: str | None = None) -> InternalMessage: + return InternalMessage(to="unused", sender=sender, body=body, thread=thread) + + +@pytest.mark.asyncio +async def test_handle_message_ignores_other_agents(agent): + msg = make_msg("unknown", "some data", None) + + await agent.handle_message(msg) + + agent.send.assert_not_called() # noqa # `agent.send` has no such property, but we mock it. + + +@pytest.mark.asyncio +async def test_handle_message_from_transcriber(agent, mock_settings): + transcription = "hello world" + msg = make_msg(mock_settings.agent_settings.transcription_name, transcription, None) + + await agent.handle_message(msg) + + agent.send.assert_awaited_once() # noqa # `agent.send` has no such property, but we mock it. + sent: InternalMessage = agent.send.call_args.args[0] # noqa + assert sent.to == mock_settings.agent_settings.bdi_belief_collector_name + assert sent.thread == "beliefs" + parsed = json.loads(sent.body) + assert parsed == {"beliefs": {"user_said": [transcription]}, "type": "belief_extraction_text"} + + +@pytest.mark.asyncio +async def test_process_user_said(agent, mock_settings): + transcription = "this is a test" + + await agent._user_said(transcription) + + agent.send.assert_awaited_once() # noqa # `agent.send` has no such property, but we mock it. + sent: InternalMessage = agent.send.call_args.args[0] # noqa + assert sent.to == mock_settings.agent_settings.bdi_belief_collector_name + assert sent.thread == "beliefs" + parsed = json.loads(sent.body) + assert parsed["beliefs"]["user_said"] == [transcription] + + +@pytest.mark.asyncio +async def test_query_llm(): + mock_response = MagicMock() + mock_response.json.return_value = { + "choices": [ + { + "message": { + "content": "null", + } + } + ] + } + mock_client = AsyncMock() + mock_client.post.return_value = mock_response + mock_async_client = MagicMock() + mock_async_client.__aenter__.return_value = mock_client + mock_async_client.__aexit__.return_value = None + + with patch( + "control_backend.agents.bdi.text_belief_extractor_agent.httpx.AsyncClient", + return_value=mock_async_client, + ): + agent = TextBeliefExtractorAgent("text_belief_agent") + + res = await agent._query_llm("hello world", {"type": "null"}) + # Response content was set as "null", so should be deserialized as None + assert res is None + + +@pytest.mark.asyncio +async def test_retry_query_llm_success(agent): + agent._query_llm.return_value = None + res = await agent._retry_query_llm("hello world", {"type": "null"}) + + agent._query_llm.assert_called_once() + assert res is None + + +@pytest.mark.asyncio +async def test_retry_query_llm_success_after_failure(agent): + agent._query_llm.side_effect = [KeyError(), "real value"] + res = await agent._retry_query_llm("hello world", {"type": "string"}) + + assert agent._query_llm.call_count == 2 + assert res == "real value" + + +@pytest.mark.asyncio +async def test_retry_query_llm_failures(agent): + agent._query_llm.side_effect = [KeyError(), KeyError(), KeyError(), "real value"] + res = await agent._retry_query_llm("hello world", {"type": "string"}) + + assert agent._query_llm.call_count == 3 + assert res is None + + +@pytest.mark.asyncio +async def test_retry_query_llm_fail_immediately(agent): + agent._query_llm.side_effect = [KeyError(), "real value"] + res = await agent._retry_query_llm("hello world", {"type": "string"}, tries=1) + + assert agent._query_llm.call_count == 1 + assert res is None + + +@pytest.mark.asyncio +async def test_extracting_beliefs_from_program(agent, sample_program): + assert len(agent.available_beliefs) == 0 + await agent.handle_message( + InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=settings.agent_settings.bdi_program_manager_name, + body=sample_program.model_dump_json(), + ), + ) + assert len(agent.available_beliefs) == 2 + + +@pytest.mark.asyncio +async def test_handle_invalid_program(agent, sample_program): + agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + assert len(agent.available_beliefs) == 2 + + await agent.handle_message( + InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=settings.agent_settings.bdi_program_manager_name, + body=json.dumps({"phases": "Invalid"}), + ), + ) + + assert len(agent.available_beliefs) == 2 + + +@pytest.mark.asyncio +async def test_handle_robot_response(agent): + initial_length = len(agent.conversation.messages) + response = "Hi, I'm Pepper. What's your name?" + + await agent.handle_message( + InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=settings.agent_settings.llm_name, + body=response, + ), + ) + + assert len(agent.conversation.messages) == initial_length + 1 + assert agent.conversation.messages[-1].role == "assistant" + assert agent.conversation.messages[-1].content == response + + +@pytest.mark.asyncio +async def test_simulated_real_turn_with_beliefs(agent, sample_program): + """Test sending user message to extract beliefs from.""" + agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + + # Send a user message with the belief that there's no more booze + agent._query_llm.return_value = {"is_pirate": None, "no_more_booze": True} + assert len(agent.conversation.messages) == 0 + await agent.handle_message( + InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=settings.agent_settings.transcription_name, + body="We're all out of schnaps.", + ), + ) + assert len(agent.conversation.messages) == 1 + + # There should be a belief set and sent to the BDI core, as well as the user_said belief + assert agent.send.call_count == 2 + + # First should be the beliefs message + message: InternalMessage = agent.send.call_args_list[0].args[0] + beliefs = BeliefMessage.model_validate_json(message.body) + assert len(beliefs.create) == 1 + assert beliefs.create[0].name == "no_more_booze" + + +@pytest.mark.asyncio +async def test_simulated_real_turn_no_beliefs(agent, sample_program): + """Test a user message to extract beliefs from, but no beliefs are formed.""" + agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + + # Send a user message with no new beliefs + agent._query_llm.return_value = {"is_pirate": None, "no_more_booze": None} + await agent.handle_message( + InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=settings.agent_settings.transcription_name, + body="Hello there!", + ), + ) + + # Only the user_said belief should've been sent + agent.send.assert_called_once() + + +@pytest.mark.asyncio +async def test_simulated_real_turn_no_new_beliefs(agent, sample_program): + """ + Test a user message to extract beliefs from, but no new beliefs are formed because they already + existed. + """ + agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + agent.beliefs["is_pirate"] = True + + # Send a user message with the belief the user is a pirate, still + agent._query_llm.return_value = {"is_pirate": True, "no_more_booze": None} + await agent.handle_message( + InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=settings.agent_settings.transcription_name, + body="Arr, nice to meet you, matey.", + ), + ) + + # Only the user_said belief should've been sent, as no beliefs have changed + agent.send.assert_called_once() + + +@pytest.mark.asyncio +async def test_simulated_real_turn_remove_belief(agent, sample_program): + """ + Test a user message to extract beliefs from, but an existing belief is determined no longer to + hold. + """ + agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + agent.beliefs["no_more_booze"] = True + + # Send a user message with the belief the user is a pirate, still + agent._query_llm.return_value = {"is_pirate": None, "no_more_booze": False} + await agent.handle_message( + InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=settings.agent_settings.transcription_name, + body="I found an untouched barrel of wine!", + ), + ) + + # Both user_said and belief change should've been sent + assert agent.send.call_count == 2 + + # Agent's current beliefs should've changed + assert not agent.beliefs["no_more_booze"] + + +@pytest.mark.asyncio +async def test_llm_failure_handling(agent, sample_program): + """ + Check that the agent handles failures gracefully without crashing. + """ + agent._query_llm.side_effect = httpx.HTTPError("") + agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + + belief_changes = await agent._infer_turn() + + assert len(belief_changes) == 0 diff --git a/test/unit/agents/bdi/test_text_extractor.py b/test/unit/agents/bdi/test_text_extractor.py deleted file mode 100644 index c51571a..0000000 --- a/test/unit/agents/bdi/test_text_extractor.py +++ /dev/null @@ -1,58 +0,0 @@ -import json -from unittest.mock import AsyncMock - -import pytest - -from control_backend.agents.bdi import ( - TextBeliefExtractorAgent, -) -from control_backend.core.agent_system import InternalMessage - - -@pytest.fixture -def agent(): - agent = TextBeliefExtractorAgent("text_belief_agent") - agent.send = AsyncMock() - return agent - - -def make_msg(sender: str, body: str, thread: str | None = None) -> InternalMessage: - return InternalMessage(to="unused", sender=sender, body=body, thread=thread) - - -@pytest.mark.asyncio -async def test_handle_message_ignores_other_agents(agent): - msg = make_msg("unknown", "some data", None) - - await agent.handle_message(msg) - - agent.send.assert_not_called() # noqa # `agent.send` has no such property, but we mock it. - - -@pytest.mark.asyncio -async def test_handle_message_from_transcriber(agent, mock_settings): - transcription = "hello world" - msg = make_msg(mock_settings.agent_settings.transcription_name, transcription, None) - - await agent.handle_message(msg) - - agent.send.assert_awaited_once() # noqa # `agent.send` has no such property, but we mock it. - sent: InternalMessage = agent.send.call_args.args[0] # noqa - assert sent.to == mock_settings.agent_settings.bdi_belief_collector_name - assert sent.thread == "beliefs" - parsed = json.loads(sent.body) - assert parsed == {"beliefs": {"user_said": [transcription]}, "type": "belief_extraction_text"} - - -@pytest.mark.asyncio -async def test_process_user_said(agent, mock_settings): - transcription = "this is a test" - - await agent._user_said(transcription) - - agent.send.assert_awaited_once() # noqa # `agent.send` has no such property, but we mock it. - sent: InternalMessage = agent.send.call_args.args[0] # noqa - assert sent.to == mock_settings.agent_settings.bdi_belief_collector_name - assert sent.thread == "beliefs" - parsed = json.loads(sent.body) - assert parsed["beliefs"]["user_said"] == [transcription] diff --git a/test/unit/schemas/test_ui_program_message.py b/test/unit/schemas/test_ui_program_message.py index a9f96dd..6014db7 100644 --- a/test/unit/schemas/test_ui_program_message.py +++ b/test/unit/schemas/test_ui_program_message.py @@ -5,11 +5,15 @@ from pydantic import ValidationError from control_backend.schemas.program import ( BasicNorm, + ConditionalNorm, Goal, + InferredBelief, KeywordBelief, + LogicalOperator, Phase, Plan, Program, + SemanticBelief, Trigger, ) @@ -97,3 +101,104 @@ def test_invalid_program(): bad = invalid_program() with pytest.raises(ValidationError): Program.model_validate(bad) + + +def test_conditional_norm_parsing(): + """ + Check that pydantic is able to preserve the type of the norm, that it doesn't lose its + "condition" field when serializing and deserializing. + """ + norm = ConditionalNorm( + name="testNormName", + id=uuid.uuid4(), + norm="testNormNorm", + critical=False, + condition=KeywordBelief( + name="testKeywordBelief", + id=uuid.uuid4(), + keyword="testKeywordBelief", + ), + ) + program = Program( + phases=[ + Phase( + name="Some phase", + id=uuid.uuid4(), + norms=[norm], + goals=[], + triggers=[], + ), + ], + ) + + parsed_program = Program.model_validate_json(program.model_dump_json()) + parsed_norm = parsed_program.phases[0].norms[0] + + assert hasattr(parsed_norm, "condition") + assert isinstance(parsed_norm, ConditionalNorm) + + +def test_belief_type_parsing(): + """ + Check that pydantic is able to discern between the different types of beliefs when serializing + and deserializing. + """ + keyword_belief = KeywordBelief( + name="testKeywordBelief", + id=uuid.uuid4(), + keyword="something", + ) + semantic_belief = SemanticBelief( + name="testSemanticBelief", + id=uuid.uuid4(), + description="something", + ) + inferred_belief = InferredBelief( + name="testInferredBelief", + id=uuid.uuid4(), + operator=LogicalOperator.OR, + left=keyword_belief, + right=semantic_belief, + ) + + program = Program( + phases=[ + Phase( + name="Some phase", + id=uuid.uuid4(), + norms=[], + goals=[], + triggers=[ + Trigger( + name="testTriggerKeywordTrigger", + id=uuid.uuid4(), + condition=keyword_belief, + plan=Plan(name="testTriggerPlanName", id=uuid.uuid4(), steps=[]), + ), + Trigger( + name="testTriggerSemanticTrigger", + id=uuid.uuid4(), + condition=semantic_belief, + plan=Plan(name="testTriggerPlanName", id=uuid.uuid4(), steps=[]), + ), + Trigger( + name="testTriggerInferredTrigger", + id=uuid.uuid4(), + condition=inferred_belief, + plan=Plan(name="testTriggerPlanName", id=uuid.uuid4(), steps=[]), + ), + ], + ), + ], + ) + + parsed_program = Program.model_validate_json(program.model_dump_json()) + + parsed_keyword_belief = parsed_program.phases[0].triggers[0].condition + assert isinstance(parsed_keyword_belief, KeywordBelief) + + parsed_semantic_belief = parsed_program.phases[0].triggers[1].condition + assert isinstance(parsed_semantic_belief, SemanticBelief) + + parsed_inferred_belief = parsed_program.phases[0].triggers[2].condition + assert isinstance(parsed_inferred_belief, InferredBelief) From 9eea4ee3454881e5a846b9eb775647d46513cab9 Mon Sep 17 00:00:00 2001 From: Kasper Date: Fri, 2 Jan 2026 12:08:20 +0100 Subject: [PATCH 17/90] feat: new ASL generation ref: N25B-376 --- src/control_backend/agents/bdi/astv2.py | 3 +- src/control_backend/agents/bdi/gen.py | 0 src/control_backend/agents/bdi/genv2.py | 354 ++++++++++++++++++++++++ 3 files changed, 356 insertions(+), 1 deletion(-) delete mode 100644 src/control_backend/agents/bdi/gen.py create mode 100644 src/control_backend/agents/bdi/genv2.py diff --git a/src/control_backend/agents/bdi/astv2.py b/src/control_backend/agents/bdi/astv2.py index f88fb6a..188b4f3 100644 --- a/src/control_backend/agents/bdi/astv2.py +++ b/src/control_backend/agents/bdi/astv2.py @@ -187,9 +187,10 @@ class StatementType(StrEnum): EMPTY = "" DO_ACTION = "." ACHIEVE_GOAL = "!" - # TEST_GOAL = "?" # TODO + TEST_GOAL = "?" ADD_BELIEF = "+" REMOVE_BELIEF = "-" + REPLACE_BELIEF = "-+" @dataclass diff --git a/src/control_backend/agents/bdi/gen.py b/src/control_backend/agents/bdi/gen.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/control_backend/agents/bdi/genv2.py b/src/control_backend/agents/bdi/genv2.py new file mode 100644 index 0000000..61980e4 --- /dev/null +++ b/src/control_backend/agents/bdi/genv2.py @@ -0,0 +1,354 @@ +import asyncio +import time +from functools import singledispatchmethod + +from slugify import slugify + +from control_backend.agents.bdi import BDICoreAgent +from control_backend.agents.bdi.astv2 import ( + AstBinaryOp, + AstExpression, + AstLiteral, + AstPlan, + AstProgram, + AstRule, + AstStatement, + AstString, + AstVar, + BinaryOperatorType, + StatementType, + TriggerType, +) +from control_backend.agents.bdi.bdi_program_manager import test_program +from control_backend.schemas.program import ( + BasicNorm, + ConditionalNorm, + GestureAction, + Goal, + InferredBelief, + KeywordBelief, + LLMAction, + LogicalOperator, + Norm, + Phase, + PlanElement, + Program, + ProgramElement, + SemanticBelief, + SpeechAction, + Trigger, +) + + +def do_things(): + program = AgentSpeakGenerator().generate(test_program) + print(program) + + +async def do_other_things(): + res = input("Wanna generate") + if res == "y": + program = AgentSpeakGenerator().generate(test_program) + filename = f"{int(time.time())}.asl" + with open(filename, "w") as f: + f.write(program) + else: + filename = "temp.asl" + bdi_agent = BDICoreAgent("BDICoreAgent", filename) + flag = asyncio.Event() + await bdi_agent.start() + await flag.wait() + + +class AgentSpeakGenerator: + _asp: AstProgram + + def generate(self, program: Program) -> str: + self._asp = AstProgram() + + self._asp.rules.append(AstRule(AstLiteral("phase", [AstString("start")]))) + self._add_keyword_inference() + self._add_response_goal() + + self._process_phases(program.phases) + + self._add_fallbacks() + + return str(self._asp) + + def _add_keyword_inference(self) -> None: + keyword = AstVar("Keyword") + message = AstVar("Message") + position = AstVar("Pos") + + self._asp.rules.append( + AstRule( + AstLiteral("keyword_said", [keyword]), + AstLiteral("user_said", [message]) + & AstLiteral(".substring", [keyword, message, position]) + & (position >= 0), + ) + ) + + def _add_response_goal(self): + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("generate_response_with_goal", [AstVar("Goal")]), + [AstLiteral("user_said", [AstVar("Message")])], + [ + AstStatement(StatementType.ADD_BELIEF, AstLiteral("responded_this_turn")), + AstStatement( + StatementType.DO_ACTION, + AstLiteral( + "findall", + [AstVar("Norm"), AstLiteral("norm", [AstVar("Norm")]), AstVar("Norms")], + ), + ), + AstStatement( + StatementType.DO_ACTION, + AstLiteral( + "reply_with_goal", [AstVar("Message"), AstVar("Norms"), AstVar("Goal")] + ), + ), + ], + ) + ) + + def _process_phases(self, phases: list[Phase]) -> None: + for curr_phase, next_phase in zip([None] + phases, phases + [None], strict=True): + if curr_phase: + self._process_phase(curr_phase) + self._add_phase_transition(curr_phase, next_phase) + + def _process_phase(self, phase: Phase) -> None: + for norm in phase.norms: + self._process_norm(norm, phase) + + self._add_default_loop(phase) + + previous_goal = None + for goal in phase.goals: + self._process_goal(goal, phase, previous_goal) + previous_goal = goal + + for trigger in phase.triggers: + self._process_trigger(trigger, phase) + + def _add_phase_transition(self, from_phase: Phase | None, to_phase: Phase | None) -> None: + from_phase_ast = ( + self._astify(from_phase) if from_phase else AstLiteral("phase", [AstString("start")]) + ) + to_phase_ast = ( + self._astify(to_phase) if to_phase else AstLiteral("phase", [AstString("end")]) + ) + + context = [from_phase_ast] + if from_phase and from_phase.goals: + context.append(self._astify(from_phase.goals[-1], achieved=True)) + + body = [ + AstStatement(StatementType.REMOVE_BELIEF, from_phase_ast), + AstStatement(StatementType.ADD_BELIEF, to_phase_ast), + ] + + if from_phase: + body.extend( + [ + AstStatement( + StatementType.TEST_GOAL, AstLiteral("user_said", [AstVar("Message")]) + ), + AstStatement( + StatementType.REPLACE_BELIEF, AstLiteral("user_said", [AstVar("Message")]) + ), + ] + ) + + self._asp.plans.append( + AstPlan(TriggerType.ADDED_GOAL, AstLiteral("transition_phase"), context, body) + ) + + def _process_norm(self, norm: Norm, phase: Phase) -> None: + rule: AstRule | None = None + + match norm: + case ConditionalNorm(condition=cond): + rule = AstRule(self._astify(norm), self._astify(phase) & self._astify(cond)) + case BasicNorm(): + rule = AstRule(self._astify(norm), self._astify(phase)) + + if not rule: + return + + self._asp.rules.append(rule) + + def _add_default_loop(self, phase: Phase) -> None: + actions = [] + + actions.append(AstStatement(StatementType.REMOVE_BELIEF, AstLiteral("responded_this_turn"))) + actions.append(AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("check_triggers"))) + + for goal in phase.goals: + actions.append(AstStatement(StatementType.ACHIEVE_GOAL, self._astify(goal))) + + actions.append(AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("transition_phase"))) + + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_BELIEF, + AstLiteral("user_said", [AstVar("Message")]), + [self._astify(phase)], + actions, + ) + ) + + def _process_goal( + self, + goal: Goal, + phase: Phase, + previous_goal: Goal | None = None, + continues_response: bool = False, + ) -> None: + context: list[AstExpression] = [self._astify(phase)] + context.append(~self._astify(goal, achieved=True)) + if previous_goal and previous_goal.can_fail: + context.append(self._astify(previous_goal, achieved=True)) + if not continues_response: + context.append(~AstLiteral("responded_this_turn")) + + body = [] + + subgoals = [] + for step in goal.plan.steps: + body.append(self._step_to_statement(step)) + if isinstance(step, Goal): + subgoals.append(step) + + if not goal.can_fail and not continues_response: + body.append(AstStatement(StatementType.ADD_BELIEF, self._astify(goal, achieved=True))) + + self._asp.plans.append(AstPlan(TriggerType.ADDED_GOAL, self._astify(goal), context, body)) + + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + self._astify(goal), + context=[], + body=[AstStatement(StatementType.EMPTY, AstLiteral("true"))], + ) + ) + + prev_goal = None + for subgoal in subgoals: + self._process_goal(subgoal, phase, prev_goal) + prev_goal = subgoal + + def _step_to_statement(self, step: PlanElement) -> AstStatement: + match step: + case Goal() as g: + return AstStatement(StatementType.ACHIEVE_GOAL, self._astify(g)) + case SpeechAction() | GestureAction() as a: + return AstStatement(StatementType.DO_ACTION, self._astify(a)) + case LLMAction() as la: + return AstStatement( + StatementType.ACHIEVE_GOAL, self._astify(la) + ) # LLM action is a goal in ASL + + # TODO: separate handling of keyword and others + def _process_trigger(self, trigger: Trigger, phase: Phase) -> None: + body = [] + subgoals = [] + + for step in trigger.plan.steps: + body.append(self._step_to_statement(step)) + if isinstance(step, Goal): + step.can_fail = False # triggers are continuous sequence + subgoals.append(step) + + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("check_triggers"), + [self._astify(phase), self._astify(trigger.condition)], + body, + ) + ) + + for subgoal in subgoals: + self._process_goal(subgoal, phase, continues_response=True) + + def _add_fallbacks(self): + # Trigger fallback + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("check_triggers"), + [], + [AstStatement(StatementType.EMPTY, AstLiteral("true"))], + ) + ) + + # Phase transition fallback + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("transition_phase"), + [], + [AstStatement(StatementType.EMPTY, AstLiteral("true"))], + ) + ) + + @singledispatchmethod + def _astify(self, element: ProgramElement) -> AstExpression: + raise NotImplementedError(f"Cannot convert element {element} to an AgentSpeak expression.") + + @_astify.register + def _(self, kwb: KeywordBelief) -> AstExpression: + return AstLiteral("keyword_said", [AstString(kwb.keyword)]) + + @_astify.register + def _(self, sb: SemanticBelief) -> AstExpression: + return AstLiteral(f"semantic_{self._slugify_str(sb.description)}") + + @_astify.register + def _(self, ib: InferredBelief) -> AstExpression: + return AstBinaryOp( + self._astify(ib.left), + BinaryOperatorType.AND if ib.operator == LogicalOperator.AND else BinaryOperatorType.OR, + self._astify(ib.right), + ) + + @_astify.register + def _(self, norm: Norm) -> AstExpression: + functor = "critical_norm" if norm.critical else "norm" + return AstLiteral(functor, [AstString(norm.norm)]) + + @_astify.register + def _(self, phase: Phase) -> AstExpression: + return AstLiteral("phase", [AstString(str(phase.id))]) + + @_astify.register + def _(self, goal: Goal, achieved: bool = False) -> AstExpression: + return AstLiteral(f"{'achieved_' if achieved else ''}{self._slugify_str(goal.name)}") + + @_astify.register + def _(self, sa: SpeechAction) -> AstExpression: + return AstLiteral("say", [AstString(sa.text)]) + + @_astify.register + def _(self, ga: GestureAction) -> AstExpression: + gesture = ga.gesture + return AstLiteral("gesture", [AstString(gesture.type), AstString(gesture.name)]) + + @_astify.register + def _(self, la: LLMAction) -> AstExpression: + return AstLiteral("generate_response_with_goal", [AstString(la.goal)]) + + @staticmethod + def _slugify_str(text: str) -> str: + return slugify(text, separator="_", stopwords=["a", "an", "the", "we", "you", "I"]) + + +if __name__ == "__main__": + # do_things() + asyncio.run(do_other_things()) From a357b6990b67a5a0f4b69050dbfb9836fea17ecc Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Tue, 6 Jan 2026 12:11:37 +0100 Subject: [PATCH 18/90] feat: send program to bdi core ref: N25B-376 --- .../bdi/{astv2.py => agentspeak_ast.py} | 0 .../bdi/{genv2.py => agentspeak_generator.py} | 31 +- .../agents/bdi/bdi_core_agent.py | 23 +- .../agents/bdi/bdi_program_manager.py | 662 +----------------- src/control_backend/main.py | 1 - .../agents/bdi/test_bdi_program_manager.py | 8 +- 6 files changed, 54 insertions(+), 671 deletions(-) rename src/control_backend/agents/bdi/{astv2.py => agentspeak_ast.py} (100%) rename src/control_backend/agents/bdi/{genv2.py => agentspeak_generator.py} (93%) diff --git a/src/control_backend/agents/bdi/astv2.py b/src/control_backend/agents/bdi/agentspeak_ast.py similarity index 100% rename from src/control_backend/agents/bdi/astv2.py rename to src/control_backend/agents/bdi/agentspeak_ast.py diff --git a/src/control_backend/agents/bdi/genv2.py b/src/control_backend/agents/bdi/agentspeak_generator.py similarity index 93% rename from src/control_backend/agents/bdi/genv2.py rename to src/control_backend/agents/bdi/agentspeak_generator.py index 61980e4..4f892e1 100644 --- a/src/control_backend/agents/bdi/genv2.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -1,11 +1,8 @@ -import asyncio -import time from functools import singledispatchmethod from slugify import slugify -from control_backend.agents.bdi import BDICoreAgent -from control_backend.agents.bdi.astv2 import ( +from control_backend.agents.bdi.agentspeak_ast import ( AstBinaryOp, AstExpression, AstLiteral, @@ -19,7 +16,6 @@ from control_backend.agents.bdi.astv2 import ( StatementType, TriggerType, ) -from control_backend.agents.bdi.bdi_program_manager import test_program from control_backend.schemas.program import ( BasicNorm, ConditionalNorm, @@ -40,26 +36,6 @@ from control_backend.schemas.program import ( ) -def do_things(): - program = AgentSpeakGenerator().generate(test_program) - print(program) - - -async def do_other_things(): - res = input("Wanna generate") - if res == "y": - program = AgentSpeakGenerator().generate(test_program) - filename = f"{int(time.time())}.asl" - with open(filename, "w") as f: - f.write(program) - else: - filename = "temp.asl" - bdi_agent = BDICoreAgent("BDICoreAgent", filename) - flag = asyncio.Event() - await bdi_agent.start() - await flag.wait() - - class AgentSpeakGenerator: _asp: AstProgram @@ -347,8 +323,3 @@ class AgentSpeakGenerator: @staticmethod def _slugify_str(text: str) -> str: return slugify(text, separator="_", stopwords=["a", "an", "the", "we", "you", "I"]) - - -if __name__ == "__main__": - # do_things() - asyncio.run(do_other_things()) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 8ff271c..7da6708 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -42,9 +42,8 @@ class BDICoreAgent(BaseAgent): bdi_agent: agentspeak.runtime.Agent - def __init__(self, name: str, asl: str): + def __init__(self, name: str): super().__init__(name) - self.asl_file = asl self.env = agentspeak.runtime.Environment() # Deep copy because we don't actually want to modify the standard actions globally self.actions = copy.deepcopy(agentspeak.stdlib.actions) @@ -69,15 +68,18 @@ class BDICoreAgent(BaseAgent): self._wake_bdi_loop.set() self.logger.debug("Setup complete.") - async def _load_asl(self): + async def _load_asl(self, file_name: str | None = None) -> None: """ Load and parse the AgentSpeak source file. """ + file_name = file_name or "src/control_backend/agents/bdi/rules.asl" + try: - with open(self.asl_file) as source: + with open(file_name) as source: self.bdi_agent = self.env.build_agent(source, self.actions) + self.logger.info(f"Loaded new ASL from {file_name}.") except FileNotFoundError: - self.logger.warning(f"Could not find the specified ASL file at {self.asl_file}.") + self.logger.warning(f"Could not find the specified ASL file at {file_name}.") self.bdi_agent = agentspeak.runtime.Agent(self.env, self.name) async def _bdi_loop(self): @@ -89,9 +91,9 @@ class BDICoreAgent(BaseAgent): the agent has deferred intentions (deadlines). """ while self._running: - # await ( - # self._wake_bdi_loop.wait() - # ) # gets set whenever there's an update to the belief base + await ( + self._wake_bdi_loop.wait() + ) # gets set whenever there's an update to the belief base # Agent knows when it's expected to have to do its next thing maybe_more_work = True @@ -116,6 +118,7 @@ class BDICoreAgent(BaseAgent): Handle incoming messages. - **Beliefs**: Updates the internal belief base. + - **Program**: Updates the internal agentspeak file to match the current program. - **LLM Responses**: Forwards the generated text to the Robot Speech Agent (actuation). :param msg: The received internal message. @@ -130,6 +133,10 @@ class BDICoreAgent(BaseAgent): self.logger.exception("Error processing belief.") return + # New agentspeak file + if msg.thread == "new_program": + await self._load_asl(msg.body) + # The message was not a belief, handle special cases based on sender match msg.sender: case settings.agent_settings.llm_name: diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 9925cfb..f8715a7 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -1,598 +1,12 @@ -import uuid -from collections.abc import Iterable - import zmq from pydantic import ValidationError -from slugify import slugify from zmq.asyncio import Context from control_backend.agents import BaseAgent +from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.config import settings -from control_backend.schemas.program import ( - Action, - BasicBelief, - BasicNorm, - Belief, - ConditionalNorm, - GestureAction, - Goal, - InferredBelief, - KeywordBelief, - LLMAction, - LogicalOperator, - Phase, - Plan, - Program, - ProgramElement, - SemanticBelief, - SpeechAction, - Trigger, -) - -test_program = Program( - phases=[ - Phase( - norms=[ - BasicNorm(norm="Talk like a pirate", id=uuid.uuid4()), - ConditionalNorm( - condition=InferredBelief( - left=KeywordBelief(keyword="Arr", id=uuid.uuid4()), - right=SemanticBelief( - description="testing", name="semantic belief", id=uuid.uuid4() - ), - operator=LogicalOperator.OR, - name="Talking to a pirate", - id=uuid.uuid4(), - ), - norm="Use nautical terms", - id=uuid.uuid4(), - ), - ConditionalNorm( - condition=SemanticBelief( - description="We are talking to a child", - name="talking to child", - id=uuid.uuid4(), - ), - norm="Do not use cuss words", - id=uuid.uuid4(), - ), - ], - triggers=[ - Trigger( - condition=InferredBelief( - left=KeywordBelief(keyword="key", id=uuid.uuid4()), - right=InferredBelief( - left=KeywordBelief(keyword="key2", id=uuid.uuid4()), - right=SemanticBelief( - description="Decode this", name="semantic belief 2", id=uuid.uuid4() - ), - operator=LogicalOperator.OR, - name="test trigger inferred inner", - id=uuid.uuid4(), - ), - operator=LogicalOperator.OR, - name="test trigger inferred outer", - id=uuid.uuid4(), - ), - plan=Plan( - steps=[ - SpeechAction(text="Testing trigger", id=uuid.uuid4()), - Goal( - name="Testing trigger", - plan=Plan( - steps=[LLMAction(goal="Do something", id=uuid.uuid4())], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - ], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ) - ], - goals=[ - Goal( - name="Determine user age", - plan=Plan( - steps=[LLMAction(goal="Determine the age of the user.", id=uuid.uuid4())], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - Goal( - name="Find the user's name", - plan=Plan( - steps=[ - Goal( - name="Greet the user", - plan=Plan( - steps=[LLMAction(goal="Greet the user.", id=uuid.uuid4())], - id=uuid.uuid4(), - ), - can_fail=False, - id=uuid.uuid4(), - ), - Goal( - name="Ask for name", - plan=Plan( - steps=[ - LLMAction(goal="Obtain the user's name.", id=uuid.uuid4()) - ], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - ], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - Goal( - name="Tell a joke", - plan=Plan( - steps=[LLMAction(goal="Tell a joke.", id=uuid.uuid4())], id=uuid.uuid4() - ), - id=uuid.uuid4(), - ), - ], - id=uuid.uuid4(), - ), - Phase( - id=uuid.uuid4(), - norms=[ - BasicNorm(norm="Use very gentle speech.", id=uuid.uuid4()), - ConditionalNorm( - condition=SemanticBelief( - description="We are talking to a child", - name="talking to child", - id=uuid.uuid4(), - ), - norm="Do not use cuss words", - id=uuid.uuid4(), - ), - ], - triggers=[ - Trigger( - condition=InferredBelief( - left=KeywordBelief(keyword="help", id=uuid.uuid4()), - right=SemanticBelief( - description="User is stuck", name="stuck", id=uuid.uuid4() - ), - operator=LogicalOperator.OR, - name="help_or_stuck", - id=uuid.uuid4(), - ), - plan=Plan( - steps=[ - Goal( - name="Unblock user", - plan=Plan( - steps=[ - LLMAction( - goal="Provide a step-by-step path to " - "resolve the user's issue.", - id=uuid.uuid4(), - ) - ], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - ], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - ], - goals=[ - Goal( - name="Clarify intent", - plan=Plan( - steps=[ - LLMAction( - goal="Ask 1-2 targeted questions to clarify the " - "user's intent, then proceed.", - id=uuid.uuid4(), - ) - ], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - Goal( - name="Provide solution", - plan=Plan( - steps=[ - LLMAction( - goal="Deliver a solution to complete the user's goal.", - id=uuid.uuid4(), - ) - ], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - Goal( - name="Summarize next steps", - plan=Plan( - steps=[ - LLMAction( - goal="Summarize what the user should do next.", id=uuid.uuid4() - ) - ], - id=uuid.uuid4(), - ), - id=uuid.uuid4(), - ), - ], - ), - ] -) - - -def do_things(): - print(AgentSpeakGenerator().generate(test_program)) - - -class AgentSpeakGenerator: - """ - Converts Pydantic representation of behavior programs into AgentSpeak(L) code string. - """ - - arrow_prefix = f"{' ' * 2}<-{' ' * 2}" - colon_prefix = f"{' ' * 2}:{' ' * 3}" - indent_prefix = " " * 6 - - def generate(self, program: Program) -> str: - lines = [] - lines.append("") - - lines += self._generate_initial_beliefs(program) - - lines += self._generate_basic_flow(program) - - lines += self._generate_phase_transitions(program) - - lines += self._generate_norms(program) - - lines += self._generate_belief_inference(program) - - lines += self._generate_goals(program) - - lines += self._generate_triggers(program) - - return "\n".join(lines) - - def _generate_initial_beliefs(self, program: Program) -> Iterable[str]: - yield "// --- Initial beliefs and agent startup ---" - - yield "phase(start)." - - yield "" - - yield "+started" - yield f"{self.colon_prefix}phase(start)" - yield f"{self.arrow_prefix}phase({program.phases[0].id if program.phases else 'end'})." - - yield from ["", ""] - - def _generate_basic_flow(self, program: Program) -> Iterable[str]: - yield "// --- Basic flow ---" - - for phase in program.phases: - yield from self._generate_basic_flow_per_phase(phase) - - yield from ["", ""] - - def _generate_basic_flow_per_phase(self, phase: Phase) -> Iterable[str]: - yield "+user_said(Message)" - yield f"{self.colon_prefix}phase({phase.id})" - - goals = phase.goals - if goals: - yield f"{self.arrow_prefix}{self._slugify(goals[0], include_prefix=True)}" - for goal in goals[1:]: - yield f"{self.indent_prefix}{self._slugify(goal, include_prefix=True)}" - - yield f"{self.indent_prefix if goals else self.arrow_prefix}!transition_phase." - - def _generate_phase_transitions(self, program: Program) -> Iterable[str]: - yield "// --- Phase transitions ---" - - if len(program.phases) == 0: - yield from ["", ""] - return - - # TODO: remove outdated things - - for i in range(-1, len(program.phases)): - predecessor = program.phases[i] if i >= 0 else None - successor = program.phases[i + 1] if i < len(program.phases) - 1 else None - yield from self._generate_phase_transition(predecessor, successor) - - yield from self._generate_phase_transition(None, None) # to avoid failing plan - - yield from ["", ""] - - def _generate_phase_transition( - self, phase: Phase | None = None, next_phase: Phase | None = None - ) -> Iterable[str]: - yield "+!transition_phase" - - if phase is None and next_phase is None: # base case true to avoid failing plan - yield f"{self.arrow_prefix}true." - return - - yield f"{self.colon_prefix}phase({phase.id if phase else 'start'})" - yield f"{self.arrow_prefix}-+phase({next_phase.id if next_phase else 'end'})." - - def _generate_norms(self, program: Program) -> Iterable[str]: - yield "// --- Norms ---" - - for phase in program.phases: - for norm in phase.norms: - if type(norm) is BasicNorm: - yield f"{self._slugify(norm)} :- phase({phase.id})." - if type(norm) is ConditionalNorm: - yield ( - f"{self._slugify(norm)} :- phase({phase.id}) & " - f"{self._slugify(norm.condition)}." - ) - - yield from ["", ""] - - def _generate_belief_inference(self, program: Program) -> Iterable[str]: - yield "// --- Belief inference rules ---" - - for phase in program.phases: - for norm in phase.norms: - if not isinstance(norm, ConditionalNorm): - continue - - yield from self._belief_inference_recursive(norm.condition) - - for trigger in phase.triggers: - yield from self._belief_inference_recursive(trigger.condition) - - yield from ["", ""] - - def _belief_inference_recursive(self, belief: Belief) -> Iterable[str]: - if type(belief) is KeywordBelief: - yield ( - f"{self._slugify(belief)} :- user_said(Message) & " - f'.substring(Message, "{belief.keyword}", Pos) & Pos >= 0.' - ) - if type(belief) is InferredBelief: - yield ( - f"{self._slugify(belief)} :- {self._slugify(belief.left)} " - f"{'&' if belief.operator == LogicalOperator.AND else '|'} " - f"{self._slugify(belief.right)}." - ) - - yield from self._belief_inference_recursive(belief.left) - yield from self._belief_inference_recursive(belief.right) - - def _generate_goals(self, program: Program) -> Iterable[str]: - yield "// --- Goals ---" - - for phase in program.phases: - previous_goal: Goal | None = None - for goal in phase.goals: - yield from self._generate_goal_plan_recursive(goal, phase, previous_goal) - previous_goal = goal - - yield from ["", ""] - - def _generate_goal_plan_recursive( - self, goal: Goal, phase: Phase, previous_goal: Goal | None = None - ) -> Iterable[str]: - yield f"+{self._slugify(goal, include_prefix=True)}" - - # Context - yield f"{self.colon_prefix}phase({phase.id}) &" - yield f"{self.indent_prefix}not responded_this_turn &" - yield f"{self.indent_prefix}not achieved_{self._slugify(goal)} &" - if previous_goal: - yield f"{self.indent_prefix}achieved_{self._slugify(previous_goal)}" - else: - yield f"{self.indent_prefix}true" - - extra_goals_to_generate = [] - - steps = goal.plan.steps - - if len(steps) == 0: - yield f"{self.arrow_prefix}true." - return - - first_step = steps[0] - yield ( - f"{self.arrow_prefix}{self._slugify(first_step, include_prefix=True)}" - f"{'.' if len(steps) == 1 and goal.can_fail else ';'}" - ) - if isinstance(first_step, Goal): - extra_goals_to_generate.append(first_step) - - for step in steps[1:-1]: - yield f"{self.indent_prefix}{self._slugify(step, include_prefix=True)};" - if isinstance(step, Goal): - extra_goals_to_generate.append(step) - - if len(steps) > 1: - last_step = steps[-1] - yield ( - f"{self.indent_prefix}{self._slugify(last_step, include_prefix=True)}" - f"{'.' if goal.can_fail else ';'}" - ) - if isinstance(last_step, Goal): - extra_goals_to_generate.append(last_step) - - if not goal.can_fail: - yield f"{self.indent_prefix}+achieved_{self._slugify(goal)}." - - yield f"+{self._slugify(goal, include_prefix=True)}" - yield f"{self.arrow_prefix}true." - - yield "" - - extra_previous_goal: Goal | None = None - for extra_goal in extra_goals_to_generate: - yield from self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) - extra_previous_goal = extra_goal - - def _generate_triggers(self, program: Program) -> Iterable[str]: - yield "// --- Triggers ---" - - for phase in program.phases: - for trigger in phase.triggers: - yield from self._generate_trigger_plan(trigger, phase) - - yield from ["", ""] - - def _generate_trigger_plan(self, trigger: Trigger, phase: Phase) -> Iterable[str]: - belief_name = self._slugify(trigger.condition) - - yield f"+{belief_name}" - yield f"{self.colon_prefix}phase({phase.id})" - - extra_goals_to_generate = [] - - steps = trigger.plan.steps - - if len(steps) == 0: - yield f"{self.arrow_prefix}true." - return - - first_step = steps[0] - yield ( - f"{self.arrow_prefix}{self._slugify(first_step, include_prefix=True)}" - f"{'.' if len(steps) == 1 else ';'}" - ) - if isinstance(first_step, Goal): - extra_goals_to_generate.append(first_step) - - for step in steps[1:-1]: - yield f"{self.indent_prefix}{self._slugify(step, include_prefix=True)};" - if isinstance(step, Goal): - extra_goals_to_generate.append(step) - - if len(steps) > 1: - last_step = steps[-1] - yield f"{self.indent_prefix}{self._slugify(last_step, include_prefix=True)}." - if isinstance(last_step, Goal): - extra_goals_to_generate.append(last_step) - - yield "" - - extra_previous_goal: Goal | None = None - for extra_goal in extra_goals_to_generate: - yield from self._generate_trigger_plan_recursive(extra_goal, phase, extra_previous_goal) - extra_previous_goal = extra_goal - - def _generate_trigger_plan_recursive( - self, goal: Goal, phase: Phase, previous_goal: Goal | None = None - ) -> Iterable[str]: - yield f"+{self._slugify(goal, include_prefix=True)}" - - extra_goals_to_generate = [] - - steps = goal.plan.steps - - if len(steps) == 0: - yield f"{self.arrow_prefix}true." - return - - first_step = steps[0] - yield ( - f"{self.arrow_prefix}{self._slugify(first_step, include_prefix=True)}" - f"{'.' if len(steps) == 1 and goal.can_fail else ';'}" - ) - if isinstance(first_step, Goal): - extra_goals_to_generate.append(first_step) - - for step in steps[1:-1]: - yield f"{self.indent_prefix}{self._slugify(step, include_prefix=True)};" - if isinstance(step, Goal): - extra_goals_to_generate.append(step) - - if len(steps) > 1: - last_step = steps[-1] - yield ( - f"{self.indent_prefix}{self._slugify(last_step, include_prefix=True)}" - f"{'.' if goal.can_fail else ';'}" - ) - if isinstance(last_step, Goal): - extra_goals_to_generate.append(last_step) - - if not goal.can_fail: - yield f"{self.indent_prefix}+achieved_{self._slugify(goal)}." - - yield f"+{self._slugify(goal, include_prefix=True)}" - yield f"{self.arrow_prefix}true." - - yield "" - - extra_previous_goal: Goal | None = None - for extra_goal in extra_goals_to_generate: - yield from self._generate_goal_plan_recursive(extra_goal, phase, extra_previous_goal) - extra_previous_goal = extra_goal - - def _slugify(self, element: ProgramElement, include_prefix: bool = False) -> str: - def base_slugify_call(text: str): - return slugify(text, separator="_", stopwords=["a", "the"]) - - if type(element) is KeywordBelief: - return f'keyword_said("{element.keyword}")' - - if type(element) is SemanticBelief: - name = element.name - return f"semantic_{base_slugify_call(name if name else element.description)}" - - if isinstance(element, BasicNorm): - return f'norm("{element.norm}")' - - if isinstance(element, Goal): - return f"{'!' if include_prefix else ''}{base_slugify_call(element.name)}" - - if isinstance(element, SpeechAction): - return f'.say("{element.text}")' - - if isinstance(element, GestureAction): - return f'.gesture("{element.gesture}")' - - if isinstance(element, LLMAction): - return f'!generate_response_with_goal("{element.goal}")' - - if isinstance(element, Action.__value__): - raise NotImplementedError( - "Have not implemented an ASL string representation for this action." - ) - - if element.name == "": - raise ValueError("Name must be initialized for this type of ProgramElement.") - - return base_slugify_call(element.name) - - def _extract_basic_beliefs_from_program(self, program: Program) -> list[BasicBelief]: - beliefs = [] - - for phase in program.phases: - for norm in phase.norms: - if isinstance(norm, ConditionalNorm): - beliefs += self._extract_basic_beliefs_from_belief(norm.condition) - - for trigger in phase.triggers: - beliefs += self._extract_basic_beliefs_from_belief(trigger.condition) - - return beliefs - - def _extract_basic_beliefs_from_belief(self, belief: Belief) -> list[BasicBelief]: - if isinstance(belief, InferredBelief): - return self._extract_basic_beliefs_from_belief( - belief.left - ) + self._extract_basic_beliefs_from_belief(belief.right) - return [belief] +from control_backend.schemas.internal_message import InternalMessage +from control_backend.schemas.program import Program class BDIProgramManager(BaseAgent): @@ -611,40 +25,36 @@ class BDIProgramManager(BaseAgent): super().__init__(**kwargs) self.sub_socket = None - # async def _send_to_bdi(self, program: Program): - # """ - # Convert a received program into BDI beliefs and send them to the BDI Core Agent. - # - # Currently, it takes the **first phase** of the program and extracts: - # - **Norms**: Constraints or rules the agent must follow. - # - **Goals**: Objectives the agent must achieve. - # - # These are sent as a ``BeliefMessage`` with ``replace=True``, meaning they will - # overwrite any existing norms/goals of the same name in the BDI agent. - # - # :param program: The program object received from the API. - # """ - # first_phase = program.phases[0] - # norms_belief = Belief( - # name="norms", - # arguments=[norm.norm for norm in first_phase.norms], - # replace=True, - # ) - # goals_belief = Belief( - # name="goals", - # arguments=[goal.description for goal in first_phase.goals], - # replace=True, - # ) - # program_beliefs = BeliefMessage(beliefs=[norms_belief, goals_belief]) - # - # message = InternalMessage( - # to=settings.agent_settings.bdi_core_name, - # sender=self.name, - # body=program_beliefs.model_dump_json(), - # thread="beliefs", - # ) - # await self.send(message) - # self.logger.debug("Sent new norms and goals to the BDI agent.") + async def _create_agentspeak_and_send_to_bdi(self, program: Program): + """ + Convert a received program into BDI beliefs and send them to the BDI Core Agent. + + Currently, it takes the **first phase** of the program and extracts: + - **Norms**: Constraints or rules the agent must follow. + - **Goals**: Objectives the agent must achieve. + + These are sent as a ``BeliefMessage`` with ``replace=True``, meaning they will + overwrite any existing norms/goals of the same name in the BDI agent. + + :param program: The program object received from the API. + """ + asg = AgentSpeakGenerator() + + asl_str = asg.generate(program) + + file_name = "src/control_backend/agents/bdi/agentspeak.asl" + + with open(file_name, "w") as f: + f.write(asl_str) + + msg = InternalMessage( + sender=self.name, + to=settings.agent_settings.bdi_core_name, + body=file_name, + thread="new_program", + ) + + await self.send(msg) async def _receive_programs(self): """ @@ -662,7 +72,7 @@ class BDIProgramManager(BaseAgent): self.logger.exception("Received an invalid program.") continue - await self._send_to_bdi(program) + await self._create_agentspeak_and_send_to_bdi(program) async def setup(self): """ @@ -678,7 +88,3 @@ class BDIProgramManager(BaseAgent): self.sub_socket.subscribe("program") self.add_behavior(self._receive_programs()) - - -if __name__ == "__main__": - do_things() diff --git a/src/control_backend/main.py b/src/control_backend/main.py index 2c8b766..d14d467 100644 --- a/src/control_backend/main.py +++ b/src/control_backend/main.py @@ -117,7 +117,6 @@ async def lifespan(app: FastAPI): BDICoreAgent, { "name": settings.agent_settings.bdi_core_name, - "asl": "src/control_backend/agents/bdi/rules.asl", }, ), "BeliefCollectorAgent": ( diff --git a/test/unit/agents/bdi/test_bdi_program_manager.py b/test/unit/agents/bdi/test_bdi_program_manager.py index a54360c..968b995 100644 --- a/test/unit/agents/bdi/test_bdi_program_manager.py +++ b/test/unit/agents/bdi/test_bdi_program_manager.py @@ -39,7 +39,7 @@ async def test_send_to_bdi(): manager.send = AsyncMock() program = Program.model_validate_json(make_valid_program_json()) - await manager._send_to_bdi(program) + await manager._create_agentspeak_and_send_to_bdi(program) assert manager.send.await_count == 1 msg: InternalMessage = manager.send.await_args[0][0] @@ -62,7 +62,7 @@ async def test_receive_programs_valid_and_invalid(): manager = BDIProgramManager(name="program_manager_test") manager.sub_socket = sub - manager._send_to_bdi = AsyncMock() + manager._create_agentspeak_and_send_to_bdi = AsyncMock() try: # Will give StopAsyncIteration when the predefined `sub.recv_multipart` side-effects run out @@ -71,7 +71,7 @@ async def test_receive_programs_valid_and_invalid(): pass # Only valid Program should have triggered _send_to_bdi - assert manager._send_to_bdi.await_count == 1 - forwarded: Program = manager._send_to_bdi.await_args[0][0] + assert manager._create_agentspeak_and_send_to_bdi.await_count == 1 + forwarded: Program = manager._create_agentspeak_and_send_to_bdi.await_args[0][0] assert forwarded.phases[0].norms[0].norm == "N1" assert forwarded.phases[0].goals[0].description == "G1" From 3406e9ac2f468b9ad2e575378007d5e0736c0ef7 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Tue, 6 Jan 2026 15:26:44 +0100 Subject: [PATCH 19/90] feat: make the pipeline work with Program and AgentSpeak ref: N25B-429 --- .../agents/bdi/agentspeak_generator.py | 80 +++++++++++++++---- .../agents/bdi/bdi_core_agent.py | 31 +++---- .../agents/bdi/belief_collector_agent.py | 2 +- .../agents/bdi/default_behavior.asl | 5 ++ src/control_backend/agents/bdi/rules.asl | 6 -- src/control_backend/agents/bdi/test.asl | 0 src/control_backend/schemas/program.py | 2 +- 7 files changed, 88 insertions(+), 38 deletions(-) create mode 100644 src/control_backend/agents/bdi/default_behavior.asl delete mode 100644 src/control_backend/agents/bdi/rules.asl delete mode 100644 src/control_backend/agents/bdi/test.asl diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 4f892e1..a446f13 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -42,9 +42,9 @@ class AgentSpeakGenerator: def generate(self, program: Program) -> str: self._asp = AstProgram() - self._asp.rules.append(AstRule(AstLiteral("phase", [AstString("start")]))) + self._asp.rules.append(AstRule(self._astify(program.phases[0]))) self._add_keyword_inference() - self._add_response_goal() + self._add_default_plans() self._process_phases(program.phases) @@ -66,11 +66,16 @@ class AgentSpeakGenerator: ) ) - def _add_response_goal(self): + def _add_default_plans(self): + self._add_reply_with_goal_plan() + self._add_say_plan() + self._add_reply_plan() + + def _add_reply_with_goal_plan(self): self._asp.plans.append( AstPlan( TriggerType.ADDED_GOAL, - AstLiteral("generate_response_with_goal", [AstVar("Goal")]), + AstLiteral("reply_with_goal", [AstVar("Goal")]), [AstLiteral("user_said", [AstVar("Message")])], [ AstStatement(StatementType.ADD_BELIEF, AstLiteral("responded_this_turn")), @@ -91,12 +96,59 @@ class AgentSpeakGenerator: ) ) + def _add_say_plan(self): + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("say", [AstVar("Text")]), + [], + [ + AstStatement(StatementType.ADD_BELIEF, AstLiteral("responded_this_turn")), + AstStatement(StatementType.DO_ACTION, AstLiteral("say", [AstVar("Text")])), + ], + ) + ) + + def _add_reply_plan(self): + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("reply"), + [AstLiteral("user_said", [AstVar("Message")])], + [ + AstStatement(StatementType.ADD_BELIEF, AstLiteral("responded_this_turn")), + AstStatement( + StatementType.DO_ACTION, + AstLiteral( + "findall", + [AstVar("Norm"), AstLiteral("norm", [AstVar("Norm")]), AstVar("Norms")], + ), + ), + AstStatement( + StatementType.DO_ACTION, + AstLiteral("reply", [AstVar("Message"), AstVar("Norms")]), + ), + ], + ) + ) + def _process_phases(self, phases: list[Phase]) -> None: for curr_phase, next_phase in zip([None] + phases, phases + [None], strict=True): if curr_phase: self._process_phase(curr_phase) self._add_phase_transition(curr_phase, next_phase) + # End phase behavior + # When deleting this, the entire `reply` plan and action can be deleted + self._asp.plans.append( + AstPlan( + type=TriggerType.ADDED_BELIEF, + trigger_literal=AstLiteral("user_said", [AstVar("Message")]), + context=[AstLiteral("phase", [AstString("end")])], + body=[AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("reply"))], + ) + ) + def _process_phase(self, phase: Phase) -> None: for norm in phase.norms: self._process_norm(norm, phase) @@ -112,14 +164,14 @@ class AgentSpeakGenerator: self._process_trigger(trigger, phase) def _add_phase_transition(self, from_phase: Phase | None, to_phase: Phase | None) -> None: - from_phase_ast = ( - self._astify(from_phase) if from_phase else AstLiteral("phase", [AstString("start")]) - ) + if from_phase is None: + return + from_phase_ast = self._astify(from_phase) to_phase_ast = ( self._astify(to_phase) if to_phase else AstLiteral("phase", [AstString("end")]) ) - context = [from_phase_ast] + context = [from_phase_ast, ~AstLiteral("responded_this_turn")] if from_phase and from_phase.goals: context.append(self._astify(from_phase.goals[-1], achieved=True)) @@ -221,14 +273,10 @@ class AgentSpeakGenerator: def _step_to_statement(self, step: PlanElement) -> AstStatement: match step: - case Goal() as g: - return AstStatement(StatementType.ACHIEVE_GOAL, self._astify(g)) - case SpeechAction() | GestureAction() as a: + case Goal() | SpeechAction() | LLMAction() as a: + return AstStatement(StatementType.ACHIEVE_GOAL, self._astify(a)) + case GestureAction() as a: return AstStatement(StatementType.DO_ACTION, self._astify(a)) - case LLMAction() as la: - return AstStatement( - StatementType.ACHIEVE_GOAL, self._astify(la) - ) # LLM action is a goal in ASL # TODO: separate handling of keyword and others def _process_trigger(self, trigger: Trigger, phase: Phase) -> None: @@ -318,7 +366,7 @@ class AgentSpeakGenerator: @_astify.register def _(self, la: LLMAction) -> AstExpression: - return AstLiteral("generate_response_with_goal", [AstString(la.goal)]) + return AstLiteral("reply_with_goal", [AstString(la.goal)]) @staticmethod def _slugify_str(text: str) -> str: diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 7da6708..249b6ee 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -48,6 +48,7 @@ class BDICoreAgent(BaseAgent): # Deep copy because we don't actually want to modify the standard actions globally self.actions = copy.deepcopy(agentspeak.stdlib.actions) self._wake_bdi_loop = asyncio.Event() + self._bdi_loop_task = None async def setup(self) -> None: """ @@ -64,7 +65,7 @@ class BDICoreAgent(BaseAgent): await self._load_asl() # Start the BDI cycle loop - self.add_behavior(self._bdi_loop()) + self._bdi_loop_task = self.add_behavior(self._bdi_loop()) self._wake_bdi_loop.set() self.logger.debug("Setup complete.") @@ -72,7 +73,7 @@ class BDICoreAgent(BaseAgent): """ Load and parse the AgentSpeak source file. """ - file_name = file_name or "src/control_backend/agents/bdi/rules.asl" + file_name = file_name or "src/control_backend/agents/bdi/default_behavior.asl" try: with open(file_name) as source: @@ -135,7 +136,10 @@ class BDICoreAgent(BaseAgent): # New agentspeak file if msg.thread == "new_program": + if self._bdi_loop_task: + self._bdi_loop_task.cancel() await self._load_asl(msg.body) + self.add_behavior(self._bdi_loop()) # The message was not a belief, handle special cases based on sender match msg.sender: @@ -246,20 +250,18 @@ class BDICoreAgent(BaseAgent): the function expects (which will be located in `term.args`). """ - @self.actions.add(".reply", 3) + @self.actions.add(".reply", 2) def _reply(agent: "BDICoreAgent", term, intention): """ Let the LLM generate a response to a user's utterance with the current norms and goals. """ message_text = agentspeak.grounded(term.args[0], intention.scope) norms = agentspeak.grounded(term.args[1], intention.scope) - goals = agentspeak.grounded(term.args[2], intention.scope) self.logger.debug("Norms: %s", norms) - self.logger.debug("Goals: %s", goals) self.logger.debug("User text: %s", message_text) - asyncio.create_task(self._send_to_llm(str(message_text), str(norms), str(goals))) + self.add_behavior(self._send_to_llm(str(message_text), str(norms), "")) yield @self.actions.add(".reply_with_goal", 3) @@ -278,7 +280,7 @@ class BDICoreAgent(BaseAgent): norms, goal, ) - # asyncio.create_task(self._send_to_llm(str(message_text), norms, str(goal))) + self.add_behavior(self._send_to_llm(str(message_text), str(norms), str(goal))) yield @self.actions.add(".say", 1) @@ -290,13 +292,14 @@ class BDICoreAgent(BaseAgent): self.logger.debug('"say" action called with text=%s', message_text) - # speech_command = SpeechCommand(data=message_text) - # speech_message = InternalMessage( - # to=settings.agent_settings.robot_speech_name, - # sender=settings.agent_settings.bdi_core_name, - # body=speech_command.model_dump_json(), - # ) - # asyncio.create_task(agent.send(speech_message)) + speech_command = SpeechCommand(data=message_text) + speech_message = InternalMessage( + to=settings.agent_settings.robot_speech_name, + sender=settings.agent_settings.bdi_core_name, + body=speech_command.model_dump_json(), + ) + # TODO: add to conversation history + self.add_behavior(self.send(speech_message)) yield @self.actions.add(".gesture", 2) diff --git a/src/control_backend/agents/bdi/belief_collector_agent.py b/src/control_backend/agents/bdi/belief_collector_agent.py index 788cff1..81c5ab2 100644 --- a/src/control_backend/agents/bdi/belief_collector_agent.py +++ b/src/control_backend/agents/bdi/belief_collector_agent.py @@ -101,7 +101,7 @@ class BDIBeliefCollectorAgent(BaseAgent): :return: A Belief object if the input is valid or None. """ try: - return Belief(name=name, arguments=arguments) + return Belief(name=name, arguments=arguments, replace=name == "user_said") except ValidationError: return None diff --git a/src/control_backend/agents/bdi/default_behavior.asl b/src/control_backend/agents/bdi/default_behavior.asl new file mode 100644 index 0000000..249689a --- /dev/null +++ b/src/control_backend/agents/bdi/default_behavior.asl @@ -0,0 +1,5 @@ +norms(""). + ++user_said(Message) : norms(Norms) <- + -user_said(Message); + .reply(Message, Norms). diff --git a/src/control_backend/agents/bdi/rules.asl b/src/control_backend/agents/bdi/rules.asl deleted file mode 100644 index cc9b4ef..0000000 --- a/src/control_backend/agents/bdi/rules.asl +++ /dev/null @@ -1,6 +0,0 @@ -norms(""). -goals(""). - -+user_said(Message) : norms(Norms) & goals(Goals) <- - -user_said(Message); - .reply(Message, Norms, Goals). diff --git a/src/control_backend/agents/bdi/test.asl b/src/control_backend/agents/bdi/test.asl deleted file mode 100644 index e69de29..0000000 diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 5a8caa9..be538b0 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -194,7 +194,7 @@ class Phase(ProgramElement): """ name: str = "" - norms: list[Norm] + norms: list[BasicNorm | ConditionalNorm] goals: list[Goal] triggers: list[Trigger] From cabe35cdbd6d29122f03c439eeaa11b7af8abb45 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Wed, 7 Jan 2026 11:44:48 +0100 Subject: [PATCH 20/90] feat: integrate AgentSpeak with semantic belief extraction ref: N25B-429 --- .../agents/bdi/agentspeak_generator.py | 8 ++- .../agents/bdi/bdi_program_manager.py | 49 ++++++++++++- .../agents/bdi/text_belief_extractor_agent.py | 70 +++++-------------- src/control_backend/schemas/belief_list.py | 14 ++++ src/control_backend/schemas/program.py | 3 +- test/unit/agents/bdi/test_bdi_core_agent.py | 6 +- .../agents/bdi/test_bdi_program_manager.py | 3 + .../agents/bdi/test_text_belief_extractor.py | 24 ++++++- .../api/v1/endpoints/test_program_endpoint.py | 2 + test/unit/schemas/test_ui_program_message.py | 1 + 10 files changed, 120 insertions(+), 60 deletions(-) create mode 100644 src/control_backend/schemas/belief_list.py diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index a446f13..8ec21df 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -332,7 +332,13 @@ class AgentSpeakGenerator: @_astify.register def _(self, sb: SemanticBelief) -> AstExpression: - return AstLiteral(f"semantic_{self._slugify_str(sb.description)}") + return AstLiteral(self.get_semantic_belief_slug(sb)) + + @staticmethod + def get_semantic_belief_slug(sb: SemanticBelief) -> str: + # If you need a method like this for other types, make a public slugify singledispatch for + # all types. + return f"semantic_{AgentSpeakGenerator._slugify_str(sb.name)}" @_astify.register def _(self, ib: InferredBelief) -> AstExpression: diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index f8715a7..54e7196 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -1,3 +1,5 @@ +import asyncio + import zmq from pydantic import ValidationError from zmq.asyncio import Context @@ -5,8 +7,9 @@ from zmq.asyncio import Context from control_backend.agents import BaseAgent from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.config import settings +from control_backend.schemas.belief_list import BeliefList from control_backend.schemas.internal_message import InternalMessage -from control_backend.schemas.program import Program +from control_backend.schemas.program import Belief, ConditionalNorm, InferredBelief, Program class BDIProgramManager(BaseAgent): @@ -56,6 +59,45 @@ class BDIProgramManager(BaseAgent): await self.send(msg) + @staticmethod + def _extract_beliefs_from_program(program: Program) -> list[Belief]: + beliefs: list[Belief] = [] + + for phase in program.phases: + for norm in phase.norms: + if isinstance(norm, ConditionalNorm): + beliefs += BDIProgramManager._extract_beliefs_from_belief(norm.condition) + + for trigger in phase.triggers: + beliefs += BDIProgramManager._extract_beliefs_from_belief(trigger.condition) + + return beliefs + + @staticmethod + def _extract_beliefs_from_belief(belief: Belief) -> list[Belief]: + if isinstance(belief, InferredBelief): + return BDIProgramManager._extract_beliefs_from_belief( + belief.left + ) + BDIProgramManager._extract_beliefs_from_belief(belief.right) + return [belief] + + async def _send_beliefs_to_semantic_belief_extractor(self, program: Program): + """ + Extract beliefs from the program and send them to the Semantic Belief Extractor Agent. + + :param program: The program received from the API. + """ + beliefs = BeliefList(beliefs=self._extract_beliefs_from_program(program)) + + message = InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=self.name, + body=beliefs.model_dump_json(), + thread="beliefs", + ) + + await self.send(message) + async def _receive_programs(self): """ Continuous loop that receives program updates from the HTTP endpoint. @@ -72,7 +114,10 @@ class BDIProgramManager(BaseAgent): self.logger.exception("Received an invalid program.") continue - await self._create_agentspeak_and_send_to_bdi(program) + await asyncio.gather( + self._create_agentspeak_and_send_to_bdi(program), + self._send_beliefs_to_semantic_belief_extractor(program), + ) async def setup(self): """ diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index 5cc75d8..c532040 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -3,21 +3,16 @@ import json import httpx from pydantic import ValidationError -from slugify import slugify from control_backend.agents.base import BaseAgent +from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings +from control_backend.schemas.belief_list import BeliefList from control_backend.schemas.belief_message import Belief as InternalBelief from control_backend.schemas.belief_message import BeliefMessage from control_backend.schemas.chat_history import ChatHistory, ChatMessage -from control_backend.schemas.program import ( - Belief, - ConditionalNorm, - InferredBelief, - Program, - SemanticBelief, -) +from control_backend.schemas.program import SemanticBelief class TextBeliefExtractorAgent(BaseAgent): @@ -32,11 +27,12 @@ class TextBeliefExtractorAgent(BaseAgent): the message itself. """ - def __init__(self, name: str): + def __init__(self, name: str, temperature: float = settings.llm_settings.code_temperature): super().__init__(name) self.beliefs: dict[str, bool] = {} self.available_beliefs: list[SemanticBelief] = [] self.conversation = ChatHistory(messages=[]) + self.temperature = temperature async def setup(self): """ @@ -85,44 +81,18 @@ class TextBeliefExtractorAgent(BaseAgent): :param msg: The received message from the program manager. """ try: - program = Program.model_validate_json(msg.body) + belief_list = BeliefList.model_validate_json(msg.body) except ValidationError: self.logger.warning( - "Received message from program manager but it is not a valid program." + "Received message from program manager but it is not a valid list of beliefs." ) return - self.logger.debug("Received a program from the program manager.") - - self.available_beliefs = self._extract_basic_beliefs_from_program(program) - - # TODO Copied from an incomplete version of the program manager. Use that one instead. - @staticmethod - def _extract_basic_beliefs_from_program(program: Program) -> list[SemanticBelief]: - beliefs = [] - - for phase in program.phases: - for norm in phase.norms: - if isinstance(norm, ConditionalNorm): - beliefs += TextBeliefExtractorAgent._extract_basic_beliefs_from_belief( - norm.condition - ) - - for trigger in phase.triggers: - beliefs += TextBeliefExtractorAgent._extract_basic_beliefs_from_belief( - trigger.condition - ) - - return beliefs - - # TODO Copied from an incomplete version of the program manager. Use that one instead. - @staticmethod - def _extract_basic_beliefs_from_belief(belief: Belief) -> list[SemanticBelief]: - if isinstance(belief, InferredBelief): - return TextBeliefExtractorAgent._extract_basic_beliefs_from_belief( - belief.left - ) + TextBeliefExtractorAgent._extract_basic_beliefs_from_belief(belief.right) - return [belief] + self.available_beliefs = [b for b in belief_list.beliefs if isinstance(b, SemanticBelief)] + self.logger.debug( + "Received %d beliefs from the program manager.", + len(self.available_beliefs), + ) async def _user_said(self, text: str): """ @@ -207,8 +177,7 @@ class TextBeliefExtractorAgent(BaseAgent): @staticmethod def _create_belief_schema(belief: SemanticBelief) -> tuple[str, dict]: - # TODO: use real belief names - return belief.name or slugify(belief.description), { + return AgentSpeakGenerator.get_semantic_belief_slug(belief), { "type": ["boolean", "null"], "description": belief.description, } @@ -237,10 +206,9 @@ class TextBeliefExtractorAgent(BaseAgent): @staticmethod def _format_beliefs(beliefs: list[SemanticBelief]): - # TODO: use real belief names return "\n".join( [ - f"- {belief.name or slugify(belief.description)}: {belief.description}" + f"- {AgentSpeakGenerator.get_semantic_belief_slug(belief)}: {belief.description}" for belief in beliefs ] ) @@ -267,7 +235,7 @@ Given the above conversation, what beliefs can be inferred? If there is no relevant information about a belief belief, give null. In case messages conflict, prefer using the most recent messages for inference. -Choose from the following list of beliefs, formatted as (belief_name, description): +Choose from the following list of beliefs, formatted as `- : `: {self._format_beliefs(beliefs)} Respond with a JSON similar to the following, but with the property names as given above: @@ -304,8 +272,7 @@ Respond with a JSON similar to the following, but with the property names as giv return None - @staticmethod - async def _query_llm(prompt: str, schema: dict) -> dict: + async def _query_llm(self, prompt: str, schema: dict) -> dict: """ Query an LLM with the given prompt and schema, return an instance of a dict conforming to that schema. @@ -333,7 +300,7 @@ Respond with a JSON similar to the following, but with the property names as giv }, }, "reasoning_effort": "low", - "temperature": settings.llm_settings.code_temperature, + "temperature": self.temperature, "stream": False, }, timeout=None, @@ -342,4 +309,5 @@ Respond with a JSON similar to the following, but with the property names as giv response_json = response.json() json_message = response_json["choices"][0]["message"]["content"] - return json.loads(json_message) + beliefs = json.loads(json_message) + return beliefs diff --git a/src/control_backend/schemas/belief_list.py b/src/control_backend/schemas/belief_list.py new file mode 100644 index 0000000..ec6a7a1 --- /dev/null +++ b/src/control_backend/schemas/belief_list.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel + +from control_backend.schemas.program import Belief as ProgramBelief + + +class BeliefList(BaseModel): + """ + Represents a list of beliefs, separated from a program. Useful in agents which need to + communicate beliefs. + + :ivar: beliefs: The list of beliefs. + """ + + beliefs: list[ProgramBelief] diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index be538b0..df20954 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -43,7 +43,6 @@ class SemanticBelief(ProgramElement): :ivar description: Description of how to form the belief, used by the LLM. """ - name: str = "" description: str @@ -113,10 +112,12 @@ class Goal(ProgramElement): for example when the achieving of the goal is dependent on the user's reply, this means that the achieved status will be set from somewhere else in the program. + :ivar description: A description of the goal, used to determine if it has been achieved. :ivar plan: The plan to execute. :ivar can_fail: Whether we can fail to achieve the goal after executing the plan. """ + description: str plan: Plan can_fail: bool = True diff --git a/test/unit/agents/bdi/test_bdi_core_agent.py b/test/unit/agents/bdi/test_bdi_core_agent.py index 2325a57..64f2ca7 100644 --- a/test/unit/agents/bdi/test_bdi_core_agent.py +++ b/test/unit/agents/bdi/test_bdi_core_agent.py @@ -20,7 +20,7 @@ def mock_agentspeak_env(): @pytest.fixture def agent(): - agent = BDICoreAgent("bdi_agent", "dummy.asl") + agent = BDICoreAgent("bdi_agent") agent.send = AsyncMock() agent.bdi_agent = MagicMock() return agent @@ -133,14 +133,14 @@ async def test_custom_actions(agent): # Invoke action mock_term = MagicMock() - mock_term.args = ["Hello", "Norm", "Goal"] + mock_term.args = ["Hello", "Norm"] mock_intention = MagicMock() # Run generator gen = action_fn(agent, mock_term, mock_intention) next(gen) # Execute - agent._send_to_llm.assert_called_with("Hello", "Norm", "Goal") + agent._send_to_llm.assert_called_with("Hello", "Norm", "") def test_add_belief_sets_event(agent): diff --git a/test/unit/agents/bdi/test_bdi_program_manager.py b/test/unit/agents/bdi/test_bdi_program_manager.py index c24b2d6..a20b058 100644 --- a/test/unit/agents/bdi/test_bdi_program_manager.py +++ b/test/unit/agents/bdi/test_bdi_program_manager.py @@ -32,6 +32,8 @@ def make_valid_program_json(norm="N1", goal="G1") -> str: Goal( id=uuid.uuid4(), name=goal, + description="This description can be used to determine whether the goal " + "has been achieved.", plan=Plan( id=uuid.uuid4(), name="Goal Plan", @@ -75,6 +77,7 @@ async def test_receive_programs_valid_and_invalid(): ] manager = BDIProgramManager(name="program_manager_test") + manager._internal_pub_socket = AsyncMock() manager.sub_socket = sub manager._create_agentspeak_and_send_to_bdi = AsyncMock() diff --git a/test/unit/agents/bdi/test_text_belief_extractor.py b/test/unit/agents/bdi/test_text_belief_extractor.py index 827adbc..176afd2 100644 --- a/test/unit/agents/bdi/test_text_belief_extractor.py +++ b/test/unit/agents/bdi/test_text_belief_extractor.py @@ -8,9 +8,11 @@ import pytest from control_backend.agents.bdi import TextBeliefExtractorAgent from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings +from control_backend.schemas.belief_list import BeliefList from control_backend.schemas.belief_message import BeliefMessage from control_backend.schemas.program import ( ConditionalNorm, + KeywordBelief, LLMAction, Phase, Plan, @@ -186,13 +188,31 @@ async def test_retry_query_llm_fail_immediately(agent): @pytest.mark.asyncio -async def test_extracting_beliefs_from_program(agent, sample_program): +async def test_extracting_semantic_beliefs(agent): + """ + The Program Manager sends beliefs to this agent. Test whether the agent handles them correctly. + """ assert len(agent.available_beliefs) == 0 + beliefs = BeliefList( + beliefs=[ + KeywordBelief( + id=uuid.uuid4(), + name="keyword_hello", + keyword="hello", + ), + SemanticBelief( + id=uuid.uuid4(), name="semantic_hello_1", description="Some semantic belief 1" + ), + SemanticBelief( + id=uuid.uuid4(), name="semantic_hello_2", description="Some semantic belief 2" + ), + ] + ) await agent.handle_message( InternalMessage( to=settings.agent_settings.text_belief_extractor_name, sender=settings.agent_settings.bdi_program_manager_name, - body=sample_program.model_dump_json(), + body=beliefs.model_dump_json(), ), ) assert len(agent.available_beliefs) == 2 diff --git a/test/unit/api/v1/endpoints/test_program_endpoint.py b/test/unit/api/v1/endpoints/test_program_endpoint.py index 379767a..c1a3fd9 100644 --- a/test/unit/api/v1/endpoints/test_program_endpoint.py +++ b/test/unit/api/v1/endpoints/test_program_endpoint.py @@ -43,6 +43,8 @@ def make_valid_program_dict(): Goal( id=uuid.uuid4(), name="Some goal", + description="This description can be used to determine whether the goal " + "has been achieved.", plan=Plan( id=uuid.uuid4(), name="Goal Plan", diff --git a/test/unit/schemas/test_ui_program_message.py b/test/unit/schemas/test_ui_program_message.py index 6014db7..6f6d5fd 100644 --- a/test/unit/schemas/test_ui_program_message.py +++ b/test/unit/schemas/test_ui_program_message.py @@ -31,6 +31,7 @@ def base_goal() -> Goal: return Goal( id=uuid.uuid4(), name="testGoalName", + description="This description can be used to determine whether the goal has been achieved.", plan=Plan( id=uuid.uuid4(), name="testGoalPlanName", From af832980c850dd87e800bb1c56c59775cdef696f Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 7 Jan 2026 12:24:46 +0100 Subject: [PATCH 21/90] feat: general slugify method ref: N25B-429 --- .../agents/bdi/agentspeak_generator.py | 24 +++++++++++++++++++ .../agents/bdi/text_belief_extractor_agent.py | 7 ++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 8ec21df..f2d7319 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -361,6 +361,10 @@ class AgentSpeakGenerator: def _(self, goal: Goal, achieved: bool = False) -> AstExpression: return AstLiteral(f"{'achieved_' if achieved else ''}{self._slugify_str(goal.name)}") + @_astify.register + def _(self, trigger: Trigger) -> AstExpression: + return AstLiteral(self.slugify(trigger)) + @_astify.register def _(self, sa: SpeechAction) -> AstExpression: return AstLiteral("say", [AstString(sa.text)]) @@ -374,6 +378,26 @@ class AgentSpeakGenerator: def _(self, la: LLMAction) -> AstExpression: return AstLiteral("reply_with_goal", [AstString(la.goal)]) + @staticmethod + @singledispatchmethod + def slugify(element: ProgramElement) -> str: + raise NotImplementedError(f"Cannot convert element {element} to a slug.") + + @staticmethod + @slugify.register + def _(sb: SemanticBelief) -> str: + return f"semantic_{AgentSpeakGenerator._slugify_str(sb.name)}" + + @staticmethod + @slugify.register + def _(g: Goal) -> str: + return AgentSpeakGenerator._slugify_str(g.name) + + @staticmethod + @slugify.register + def _(t: Trigger): + return f"trigger_{AgentSpeakGenerator._slugify_str(t.name)}" + @staticmethod def _slugify_str(text: str) -> str: return slugify(text, separator="_", stopwords=["a", "an", "the", "we", "you", "I"]) diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index c532040..37af8b4 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -177,7 +177,7 @@ class TextBeliefExtractorAgent(BaseAgent): @staticmethod def _create_belief_schema(belief: SemanticBelief) -> tuple[str, dict]: - return AgentSpeakGenerator.get_semantic_belief_slug(belief), { + return AgentSpeakGenerator.slugify(belief), { "type": ["boolean", "null"], "description": belief.description, } @@ -207,10 +207,7 @@ class TextBeliefExtractorAgent(BaseAgent): @staticmethod def _format_beliefs(beliefs: list[SemanticBelief]): return "\n".join( - [ - f"- {AgentSpeakGenerator.get_semantic_belief_slug(belief)}: {belief.description}" - for belief in beliefs - ] + [f"- {AgentSpeakGenerator.slugify(belief)}: {belief.description}" for belief in beliefs] ) async def _infer_beliefs( From 07d70cb781cecd6cc509228343cbfed91f0bcb13 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 7 Jan 2026 13:02:23 +0100 Subject: [PATCH 22/90] fix: single dispatch order ref: N25B-429 --- src/control_backend/agents/bdi/agentspeak_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index f2d7319..1c313ce 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -378,23 +378,23 @@ class AgentSpeakGenerator: def _(self, la: LLMAction) -> AstExpression: return AstLiteral("reply_with_goal", [AstString(la.goal)]) - @staticmethod @singledispatchmethod + @staticmethod def slugify(element: ProgramElement) -> str: raise NotImplementedError(f"Cannot convert element {element} to a slug.") - @staticmethod @slugify.register + @staticmethod def _(sb: SemanticBelief) -> str: return f"semantic_{AgentSpeakGenerator._slugify_str(sb.name)}" - @staticmethod @slugify.register + @staticmethod def _(g: Goal) -> str: return AgentSpeakGenerator._slugify_str(g.name) - @staticmethod @slugify.register + @staticmethod def _(t: Trigger): return f"trigger_{AgentSpeakGenerator._slugify_str(t.name)}" From 324a63e5cc60437aa7cd9f868bfe30a381070614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Otgaar?= Date: Wed, 7 Jan 2026 14:45:42 +0100 Subject: [PATCH 23/90] chore: add styles to user_interrupt_agent --- .../user_interrupt/user_interrupt_agent.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index b2efc41..8a4d2a2 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -71,6 +71,16 @@ class UserInterruptAgent(BaseAgent): "Forwarded button press (override) with context '%s' to BDIProgramManager.", event_context, ) + + elif event_type == "next_phase": + _ = 1 + + elif event_type == "reset_phase": + _ = 1 + + elif event_type == " reset_experiment": + _ = 1 + else: self.logger.warning( "Received button press with unknown type '%s' (context: '%s').", @@ -78,6 +88,15 @@ class UserInterruptAgent(BaseAgent): event_context, ) + async def _send_experiment_control_to_bdi_core(self, type): + out_msg = InternalMessage( + to=settings.agent_settings.bdi_core_name, + sender=self.name, + thread=type, + body="", + ) + await self.send(out_msg) + async def _send_to_speech_agent(self, text_to_say: str): """ method to send prioritized speech command to RobotSpeechAgent. From 34afca6652dde4fbb4aa993d97454f11225ef5a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Otgaar?= Date: Wed, 7 Jan 2026 15:07:33 +0100 Subject: [PATCH 24/90] chore: automatically send the experiment controls to the bdi core in the user interupt agent. --- .../user_interrupt/user_interrupt_agent.py | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 8a4d2a2..e58a42b 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -72,14 +72,8 @@ class UserInterruptAgent(BaseAgent): event_context, ) - elif event_type == "next_phase": - _ = 1 - - elif event_type == "reset_phase": - _ = 1 - - elif event_type == " reset_experiment": - _ = 1 + elif event_type in ["next_phase", "reset_phase", "reset_experiment"]: + await self._send_experiment_control_to_bdi_core(event_type) else: self.logger.warning( @@ -89,12 +83,33 @@ class UserInterruptAgent(BaseAgent): ) async def _send_experiment_control_to_bdi_core(self, type): + """ + method to send experiment control buttons to bdi core. + + :param type: the type of control button we should send to the bdi core. + """ + # Switch which thread we should send to bdi core + thread = "" + match type: + case "next_phase": + thread = "force_next_phase" + case "reset_phase": + thread = "reset_current_phase" + case "reset_experiment": + thread = "reset_experiment" + case _: + self.logger.warning( + "Received unknown experiment control type '%s' to send to BDI Core.", + type, + ) + out_msg = InternalMessage( to=settings.agent_settings.bdi_core_name, sender=self.name, - thread=type, + thread=thread, body="", ) + self.logger.debug("Sending experiment control '%s' to BDI Core.", thread) await self.send(out_msg) async def _send_to_speech_agent(self, text_to_say: str): From 3189b9fee34ec373779256755c7c6fc4067f1774 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Wed, 7 Jan 2026 15:19:23 +0100 Subject: [PATCH 25/90] fix: let belief extractor send user_said belief ref: N25B-429 --- .../agents/bdi/text_belief_extractor_agent.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index 37af8b4..800d5e4 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -100,13 +100,12 @@ class TextBeliefExtractorAgent(BaseAgent): :param text: User's transcribed text. """ - belief = {"beliefs": {"user_said": [text]}, "type": "belief_extraction_text"} - payload = json.dumps(belief) - belief_msg = InternalMessage( - to=settings.agent_settings.bdi_belief_collector_name, + to=settings.agent_settings.bdi_core_name, sender=self.name, - body=payload, + body=BeliefMessage( + replace=[InternalBelief(name="user_said", arguments=[text])], + ).model_dump_json(), thread="beliefs", ) await self.send(belief_msg) From 76dfcb23ef3f5777877eeb2bcaf4a73a0858297d Mon Sep 17 00:00:00 2001 From: Storm Date: Wed, 7 Jan 2026 16:03:49 +0100 Subject: [PATCH 26/90] feat: added pause functionality ref: N25B-350 --- .../agents/perception/vad_agent.py | 41 ++++++++++++++++ .../user_interrupt/user_interrupt_agent.py | 47 ++++++++++++++++++- src/control_backend/schemas/ri_message.py | 11 +++++ 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/src/control_backend/agents/perception/vad_agent.py b/src/control_backend/agents/perception/vad_agent.py index 8ccff0a..320a849 100644 --- a/src/control_backend/agents/perception/vad_agent.py +++ b/src/control_backend/agents/perception/vad_agent.py @@ -7,6 +7,7 @@ import zmq.asyncio as azmq from control_backend.agents import BaseAgent from control_backend.core.config import settings +from control_backend.schemas.internal_message import InternalMessage from ...schemas.program_status import PROGRAM_STATUS, ProgramStatus from .transcription_agent.transcription_agent import TranscriptionAgent @@ -86,6 +87,12 @@ class VADAgent(BaseAgent): self.audio_buffer = np.array([], dtype=np.float32) self.i_since_speech = settings.behaviour_settings.vad_initial_since_speech self._ready = asyncio.Event() + + # Pause control + self._reset_needed = False + self._paused = asyncio.Event() + self._paused.set() # Not paused at start + self.model = None async def setup(self): @@ -213,6 +220,16 @@ class VADAgent(BaseAgent): """ await self._ready.wait() while self._running: + await self._paused.wait() + + # After being unpaused, reset stream and buffers + if self._reset_needed: + self.logger.debug("Resuming: resetting stream and buffers.") + await self._reset_stream() + self.audio_buffer = np.array([], dtype=np.float32) + self.i_since_speech = settings.behaviour_settings.vad_initial_since_speech + self._reset_needed = False + assert self.audio_in_poller is not None data = await self.audio_in_poller.poll() if data is None: @@ -254,3 +271,27 @@ class VADAgent(BaseAgent): # At this point, we know that the speech has ended. # Prepend the last chunk that had no speech, for a more fluent boundary self.audio_buffer = chunk + +async def handle_message(self, msg: InternalMessage): + """ + Handle incoming messages. + + Expects messages to pause or resume the VAD processing from User Interrupt Agent. + + :param msg: The received internal message. + """ + sender = msg.sender + + if sender == settings.agent_settings.user_interrupt_name: + if msg.body == "PAUSE": + self.logger.info("Pausing VAD processing.") + self._paused.clear() + # If the robot needs to pick up speaking where it left off, do not set _reset_needed + self._reset_needed = True + elif msg.body == "RESUME": + self.logger.info("Resuming VAD processing.") + self._paused.set() + else: + self.logger.warning(f"Unknown command from User Interrupt Agent: {msg.body}") + else: + self.logger.debug(f"Ignoring message from unknown sender: {sender}") diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index e58a42b..842231a 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -6,7 +6,12 @@ from zmq.asyncio import Context from control_backend.agents import BaseAgent from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings -from control_backend.schemas.ri_message import GestureCommand, RIEndpoint, SpeechCommand +from control_backend.schemas.ri_message import ( + GestureCommand, + PauseCommand, + RIEndpoint, + SpeechCommand, +) class UserInterruptAgent(BaseAgent): @@ -71,7 +76,12 @@ class UserInterruptAgent(BaseAgent): "Forwarded button press (override) with context '%s' to BDIProgramManager.", event_context, ) - + elif event_type == "pause": + await self._send_pause_command(event_context) + if event_context: + self.logger.info("Sent pause command.") + else: + self.logger.info("Sent resume command.") elif event_type in ["next_phase", "reset_phase", "reset_experiment"]: await self._send_experiment_control_to_bdi_core(event_type) @@ -163,6 +173,39 @@ class UserInterruptAgent(BaseAgent): "Sent button_override belief with id '%s' to Program manager.", belief_id, ) + + async def _send_pause_command(self, pause : bool): + """ + Send a pause command to the Robot Interface via the RI Communication Agent. + Send a pause command to the other internal agents; for now just VAD agent. + """ + cmd = PauseCommand(data=pause) + message = InternalMessage( + to=settings.agent_settings.ri_communication_name, + sender=self.name, + body=cmd.model_dump_json(), + ) + await self.send(message) + + if pause: + # Send pause to VAD agent + vad_message = InternalMessage( + to=settings.agent_settings.vad_name, + sender=self.name, + body="PAUSE", + ) + await self.send(vad_message) + self.logger.info("Sent pause command to VAD Agent and RI Communication Agent.") + else: + # Send resume to VAD agent + vad_message = InternalMessage( + to=settings.agent_settings.vad_name, + sender=self.name, + body="RESUME", + ) + await self.send(vad_message) + self.logger.info("Sent resume command to VAD Agent and RI Communication Agent.") + async def setup(self): """ diff --git a/src/control_backend/schemas/ri_message.py b/src/control_backend/schemas/ri_message.py index a48dec6..7c1ef22 100644 --- a/src/control_backend/schemas/ri_message.py +++ b/src/control_backend/schemas/ri_message.py @@ -64,3 +64,14 @@ class GestureCommand(RIMessage): if self.endpoint not in allowed: raise ValueError("endpoint must be GESTURE_SINGLE or GESTURE_TAG") return self + +class PauseCommand(RIMessage): + """ + A specific command to pause or unpause the robot's actions. + + :ivar endpoint: Fixed to ``RIEndpoint.PAUSE``. + :ivar data: A boolean indicating whether to pause (True) or unpause (False). + """ + + endpoint: RIEndpoint = RIEndpoint(RIEndpoint.PAUSE) + data: bool From aa5b386f658415e52f22b4f0390e67974f9770d1 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Wed, 7 Jan 2026 17:08:23 +0100 Subject: [PATCH 27/90] feat: semantically determine goal completion ref: N25B-432 --- .../agents/bdi/bdi_core_agent.py | 13 +- .../agents/bdi/bdi_program_manager.py | 64 ++- .../agents/bdi/belief_collector_agent.py | 2 +- .../agents/bdi/text_belief_extractor_agent.py | 443 ++++++++++++------ src/control_backend/core/agent_system.py | 11 +- src/control_backend/schemas/belief_list.py | 5 + src/control_backend/schemas/belief_message.py | 3 + src/control_backend/schemas/program.py | 2 +- 8 files changed, 380 insertions(+), 163 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 58ece29..3baa493 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -205,12 +205,15 @@ class BDICoreAgent(BaseAgent): self.logger.debug(f"Added belief {self.format_belief_string(name, args)}") - def _remove_belief(self, name: str, args: Iterable[str]): + def _remove_belief(self, name: str, args: Iterable[str] | None): """ Removes a specific belief (with arguments), if it exists. """ - new_args = (agentspeak.Literal(arg) for arg in args) - term = agentspeak.Literal(name, new_args) + if args is None: + term = agentspeak.Literal(name) + else: + new_args = (agentspeak.Literal(arg) for arg in args) + term = agentspeak.Literal(name, new_args) result = self.bdi_agent.call( agentspeak.Trigger.removal, @@ -346,8 +349,8 @@ class BDICoreAgent(BaseAgent): self.logger.info("Message sent to LLM agent: %s", text) @staticmethod - def format_belief_string(name: str, args: Iterable[str] = []): + def format_belief_string(name: str, args: Iterable[str] | None = []): """ Given a belief's name and its args, return a string of the form "name(*args)" """ - return f"{name}{'(' if args else ''}{','.join(args)}{')' if args else ''}" + return f"{name}{'(' if args else ''}{','.join(args or [])}{')' if args else ''}" diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 54e7196..96d924d 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -7,9 +7,9 @@ from zmq.asyncio import Context from control_backend.agents import BaseAgent from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.config import settings -from control_backend.schemas.belief_list import BeliefList +from control_backend.schemas.belief_list import BeliefList, GoalList from control_backend.schemas.internal_message import InternalMessage -from control_backend.schemas.program import Belief, ConditionalNorm, InferredBelief, Program +from control_backend.schemas.program import Belief, ConditionalNorm, Goal, InferredBelief, Program class BDIProgramManager(BaseAgent): @@ -63,24 +63,23 @@ class BDIProgramManager(BaseAgent): def _extract_beliefs_from_program(program: Program) -> list[Belief]: beliefs: list[Belief] = [] + def extract_beliefs_from_belief(belief: Belief) -> list[Belief]: + if isinstance(belief, InferredBelief): + return extract_beliefs_from_belief(belief.left) + extract_beliefs_from_belief( + belief.right + ) + return [belief] + for phase in program.phases: for norm in phase.norms: if isinstance(norm, ConditionalNorm): - beliefs += BDIProgramManager._extract_beliefs_from_belief(norm.condition) + beliefs += extract_beliefs_from_belief(norm.condition) for trigger in phase.triggers: - beliefs += BDIProgramManager._extract_beliefs_from_belief(trigger.condition) + beliefs += extract_beliefs_from_belief(trigger.condition) return beliefs - @staticmethod - def _extract_beliefs_from_belief(belief: Belief) -> list[Belief]: - if isinstance(belief, InferredBelief): - return BDIProgramManager._extract_beliefs_from_belief( - belief.left - ) + BDIProgramManager._extract_beliefs_from_belief(belief.right) - return [belief] - async def _send_beliefs_to_semantic_belief_extractor(self, program: Program): """ Extract beliefs from the program and send them to the Semantic Belief Extractor Agent. @@ -98,6 +97,46 @@ class BDIProgramManager(BaseAgent): await self.send(message) + @staticmethod + def _extract_goals_from_program(program: Program) -> list[Goal]: + """ + Extract all goals from the program, including subgoals. + + :param program: The program received from the API. + :return: A list of Goal objects. + """ + goals: list[Goal] = [] + + def extract_goals_from_goal(goal_: Goal) -> list[Goal]: + goals_: list[Goal] = [goal] + for plan in goal_.plan: + if isinstance(plan, Goal): + goals_.extend(extract_goals_from_goal(plan)) + return goals_ + + for phase in program.phases: + for goal in phase.goals: + goals.extend(extract_goals_from_goal(goal)) + + return goals + + async def _send_goals_to_semantic_belief_extractor(self, program: Program): + """ + Extract goals from the program and send them to the Semantic Belief Extractor Agent. + + :param program: The program received from the API. + """ + goals = GoalList(goals=self._extract_goals_from_program(program)) + + message = InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + sender=self.name, + body=goals.model_dump_json(), + thread="goals", + ) + + await self.send(message) + async def _receive_programs(self): """ Continuous loop that receives program updates from the HTTP endpoint. @@ -117,6 +156,7 @@ class BDIProgramManager(BaseAgent): await asyncio.gather( self._create_agentspeak_and_send_to_bdi(program), self._send_beliefs_to_semantic_belief_extractor(program), + self._send_goals_to_semantic_belief_extractor(program), ) async def setup(self): diff --git a/src/control_backend/agents/bdi/belief_collector_agent.py b/src/control_backend/agents/bdi/belief_collector_agent.py index 6f89d2a..ac0e2e5 100644 --- a/src/control_backend/agents/bdi/belief_collector_agent.py +++ b/src/control_backend/agents/bdi/belief_collector_agent.py @@ -101,7 +101,7 @@ class BDIBeliefCollectorAgent(BaseAgent): :return: A Belief object if the input is valid or None. """ try: - return Belief(name=name, arguments=arguments, replace=name == "user_said") + return Belief(name=name, arguments=arguments) except ValidationError: return None diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index 800d5e4..7e3570f 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -2,17 +2,45 @@ import asyncio import json import httpx -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from control_backend.agents.base import BaseAgent from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings -from control_backend.schemas.belief_list import BeliefList +from control_backend.schemas.belief_list import BeliefList, GoalList from control_backend.schemas.belief_message import Belief as InternalBelief from control_backend.schemas.belief_message import BeliefMessage from control_backend.schemas.chat_history import ChatHistory, ChatMessage -from control_backend.schemas.program import SemanticBelief +from control_backend.schemas.program import Goal, SemanticBelief + +type JSONLike = None | bool | int | float | str | list["JSONLike"] | dict[str, "JSONLike"] + + +class BeliefState(BaseModel): + true: set[InternalBelief] = set() + false: set[InternalBelief] = set() + + def difference(self, other: "BeliefState") -> "BeliefState": + return BeliefState( + true=self.true - other.true, + false=self.false - other.false, + ) + + def union(self, other: "BeliefState") -> "BeliefState": + return BeliefState( + true=self.true | other.true, + false=self.false | other.false, + ) + + def __sub__(self, other): + return self.difference(other) + + def __or__(self, other): + return self.union(other) + + def __bool__(self): + return bool(self.true) or bool(self.false) class TextBeliefExtractorAgent(BaseAgent): @@ -27,12 +55,14 @@ class TextBeliefExtractorAgent(BaseAgent): the message itself. """ - def __init__(self, name: str, temperature: float = settings.llm_settings.code_temperature): + def __init__(self, name: str): super().__init__(name) - self.beliefs: dict[str, bool] = {} - self.available_beliefs: list[SemanticBelief] = [] + self._llm = self.LLM(self, settings.llm_settings.n_parallel) + self.belief_inferrer = SemanticBeliefInferrer(self._llm) + self.goal_inferrer = GoalAchievementInferrer(self._llm) + self._current_beliefs = BeliefState() + self._current_goal_completions: dict[str, bool] = {} self.conversation = ChatHistory(messages=[]) - self.temperature = temperature async def setup(self): """ @@ -53,8 +83,9 @@ class TextBeliefExtractorAgent(BaseAgent): case settings.agent_settings.transcription_name: self.logger.debug("Received text from transcriber: %s", msg.body) self._apply_conversation_message(ChatMessage(role="user", content=msg.body)) - await self._infer_new_beliefs() await self._user_said(msg.body) + await self._infer_new_beliefs() + await self._infer_goal_completions() case settings.agent_settings.llm_name: self.logger.debug("Received text from LLM: %s", msg.body) self._apply_conversation_message(ChatMessage(role="assistant", content=msg.body)) @@ -76,10 +107,19 @@ class TextBeliefExtractorAgent(BaseAgent): def _handle_program_manager_message(self, msg: InternalMessage): """ - Handle a message from the program manager: extract available beliefs from it. + Handle a message from the program manager: extract available beliefs and goals from it. :param msg: The received message from the program manager. """ + match msg.thread: + case "beliefs": + self._handle_beliefs_message(msg) + case "goals": + self._handle_goals_message(msg) + case _: + self.logger.warning("Received unexpected message from %s", msg.sender) + + def _handle_beliefs_message(self, msg: InternalMessage): try: belief_list = BeliefList.model_validate_json(msg.body) except ValidationError: @@ -88,10 +128,28 @@ class TextBeliefExtractorAgent(BaseAgent): ) return - self.available_beliefs = [b for b in belief_list.beliefs if isinstance(b, SemanticBelief)] + available_beliefs = [b for b in belief_list.beliefs if isinstance(b, SemanticBelief)] + self.belief_inferrer.available_beliefs = available_beliefs self.logger.debug( - "Received %d beliefs from the program manager.", - len(self.available_beliefs), + "Received %d semantic beliefs from the program manager.", + len(available_beliefs), + ) + + def _handle_goals_message(self, msg: InternalMessage): + try: + goals_list = GoalList.model_validate_json(msg.body) + except ValidationError: + self.logger.warning( + "Received message from program manager but it is not a valid list of goals." + ) + return + + # Use only goals that can fail, as the others are always assumed to be completed + available_goals = [g for g in goals_list.goals if g.can_fail] + self.goal_inferrer.goals = available_goals + self.logger.debug( + "Received %d failable goals from the program manager.", + len(available_goals), ) async def _user_said(self, text: str): @@ -111,109 +169,199 @@ class TextBeliefExtractorAgent(BaseAgent): await self.send(belief_msg) async def _infer_new_beliefs(self): - """ - Process conversation history to extract beliefs, semantically. Any changed beliefs are sent - to the BDI core. - """ - # Return instantly if there are no beliefs to infer - if not self.available_beliefs: + conversation_beliefs = await self.belief_inferrer.infer_from_conversation(self.conversation) + + new_beliefs = conversation_beliefs - self._current_beliefs + if not new_beliefs: return - candidate_beliefs = await self._infer_turn() - belief_changes = BeliefMessage() - for belief_key, belief_value in candidate_beliefs.items(): - if belief_value is None: - continue - old_belief_value = self.beliefs.get(belief_key) - if belief_value == old_belief_value: - continue + self._current_beliefs |= new_beliefs - self.beliefs[belief_key] = belief_value + belief_changes = BeliefMessage( + create=list(new_beliefs.true), + delete=list(new_beliefs.false), + ) - belief = InternalBelief(name=belief_key, arguments=None) - if belief_value: - belief_changes.create.append(belief) - else: - belief_changes.delete.append(belief) - - # Return if there were no changes in beliefs - if not belief_changes.has_values(): - return - - beliefs_message = InternalMessage( + message = InternalMessage( to=settings.agent_settings.bdi_core_name, sender=self.name, body=belief_changes.model_dump_json(), thread="beliefs", ) - await self.send(beliefs_message) + await self.send(message) - @staticmethod - def _split_into_chunks[T](items: list[T], n: int) -> list[list[T]]: - k, m = divmod(len(items), n) - return [items[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] + async def _infer_goal_completions(self): + goal_completions = await self.goal_inferrer.infer_from_conversation(self.conversation) - async def _infer_turn(self) -> dict: + new_achieved = [ + InternalBelief(name=goal, arguments=None) + for goal, achieved in goal_completions.items() + if achieved and self._current_goal_completions.get(goal) != achieved + ] + new_not_achieved = [ + InternalBelief(name=goal, arguments=None) + for goal, achieved in goal_completions.items() + if not achieved and self._current_goal_completions.get(goal) != achieved + ] + for goal, achieved in goal_completions.items(): + self._current_goal_completions[goal] = achieved + + if not new_achieved and not new_not_achieved: + return + + belief_changes = BeliefMessage( + create=new_achieved, + delete=new_not_achieved, + ) + message = InternalMessage( + to=settings.agent_settings.bdi_core_name, + sender=self.name, + body=belief_changes.model_dump_json(), + thread="beliefs", + ) + await self.send(message) + + class LLM: """ - Process the stored conversation history to extract semantic beliefs. Returns a list of - beliefs that have been set to ``True``, ``False`` or ``None``. - - :return: A dict mapping belief names to a value ``True``, ``False`` or ``None``. + Class that handles sending structured generation requests to an LLM. """ + + def __init__(self, agent: "TextBeliefExtractorAgent", n_parallel: int): + self._agent = agent + self._semaphore = asyncio.Semaphore(n_parallel) + + async def query(self, prompt: str, schema: dict, tries: int = 3) -> JSONLike | None: + """ + Query the LLM with the given prompt and schema, return an instance of a dict conforming + to this schema. Try ``tries`` times, or return None. + + :param prompt: Prompt to be queried. + :param schema: Schema to be queried. + :param tries: Number of times to try to query the LLM. + :return: An instance of a dict conforming to this schema, or None if failed. + """ + try_count = 0 + while try_count < tries: + try_count += 1 + + try: + return await self._query_llm(prompt, schema) + except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e: + if try_count < tries: + continue + self._agent.logger.exception( + "Failed to get LLM response after %d tries.", + try_count, + exc_info=e, + ) + + return None + + async def _query_llm(self, prompt: str, schema: dict) -> JSONLike: + """ + Query an LLM with the given prompt and schema, return an instance of a dict conforming + to that schema. + + :param prompt: The prompt to be queried. + :param schema: Schema to use during response. + :return: A dict conforming to this schema. + :raises httpx.HTTPStatusError: If the LLM server responded with an error. + :raises json.JSONDecodeError: If the LLM response was not valid JSON. May happen if the + response was cut off early due to length limitations. + :raises KeyError: If the LLM server responded with no error, but the response was + invalid. + """ + async with self._semaphore: + async with httpx.AsyncClient() as client: + response = await client.post( + settings.llm_settings.local_llm_url, + json={ + "model": settings.llm_settings.local_llm_model, + "messages": [{"role": "user", "content": prompt}], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "Beliefs", + "strict": True, + "schema": schema, + }, + }, + "reasoning_effort": "low", + "temperature": settings.llm_settings.code_temperature, + "stream": False, + }, + timeout=30.0, + ) + response.raise_for_status() + + response_json = response.json() + json_message = response_json["choices"][0]["message"]["content"] + return json.loads(json_message) + + +class SemanticBeliefInferrer: + """ + Class that handles only prompting an LLM for semantic beliefs. + """ + + def __init__( + self, + llm: "TextBeliefExtractorAgent.LLM", + available_beliefs: list[SemanticBelief] | None = None, + ): + self._llm = llm + self.available_beliefs: list[SemanticBelief] = available_beliefs or [] + + async def infer_from_conversation(self, conversation: ChatHistory) -> BeliefState: + """ + Process conversation history to extract beliefs, semantically. The result is an object that + describes all beliefs that hold or don't hold based on the full conversation. + + :param conversation: The conversation history to be processed. + :return: An object that describes beliefs. + """ + # Return instantly if there are no beliefs to infer + if not self.available_beliefs: + return BeliefState() + n_parallel = max(1, min(settings.llm_settings.n_parallel - 1, len(self.available_beliefs))) - all_beliefs = await asyncio.gather( + all_beliefs: list[dict[str, bool | None] | None] = await asyncio.gather( *[ - self._infer_beliefs(self.conversation, beliefs) + self._infer_beliefs(conversation, beliefs) for beliefs in self._split_into_chunks(self.available_beliefs, n_parallel) ] ) - retval = {} + retval = BeliefState() for beliefs in all_beliefs: if beliefs is None: continue - retval.update(beliefs) + for belief_name, belief_holds in beliefs.items(): + if belief_holds is None: + continue + belief = InternalBelief(name=belief_name, arguments=None) + if belief_holds: + retval.true.add(belief) + else: + retval.false.add(belief) return retval @staticmethod - def _create_belief_schema(belief: SemanticBelief) -> tuple[str, dict]: - return AgentSpeakGenerator.slugify(belief), { - "type": ["boolean", "null"], - "description": belief.description, - } + def _split_into_chunks[T](items: list[T], n: int) -> list[list[T]]: + """ + Split a list into ``n`` chunks, making each chunk approximately ``len(items) / n`` long. - @staticmethod - def _create_beliefs_schema(beliefs: list[SemanticBelief]) -> dict: - belief_schemas = [ - TextBeliefExtractorAgent._create_belief_schema(belief) for belief in beliefs - ] - - return { - "type": "object", - "properties": dict(belief_schemas), - "required": [name for name, _ in belief_schemas], - } - - @staticmethod - def _format_message(message: ChatMessage): - return f"{message.role.upper()}:\n{message.content}" - - @staticmethod - def _format_conversation(conversation: ChatHistory): - return "\n\n".join( - [TextBeliefExtractorAgent._format_message(message) for message in conversation.messages] - ) - - @staticmethod - def _format_beliefs(beliefs: list[SemanticBelief]): - return "\n".join( - [f"- {AgentSpeakGenerator.slugify(belief)}: {belief.description}" for belief in beliefs] - ) + :param items: The list of items to split. + :param n: The number of desired chunks. + :return: A list of chunks each approximately ``len(items) / n`` long. + """ + k, m = divmod(len(items), n) + return [items[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] async def _infer_beliefs( self, conversation: ChatHistory, beliefs: list[SemanticBelief], - ) -> dict | None: + ) -> dict[str, bool | None] | None: """ Infer given beliefs based on the given conversation. :param conversation: The conversation to infer beliefs from. @@ -240,70 +388,79 @@ Respond with a JSON similar to the following, but with the property names as giv schema = self._create_beliefs_schema(beliefs) - return await self._retry_query_llm(prompt, schema) + return await self._llm.query(prompt, schema) - async def _retry_query_llm(self, prompt: str, schema: dict, tries: int = 3) -> dict | None: + @staticmethod + def _create_belief_schema(belief: SemanticBelief) -> tuple[str, dict]: + return AgentSpeakGenerator.slugify(belief), { + "type": ["boolean", "null"], + "description": belief.description, + } + + @staticmethod + def _create_beliefs_schema(beliefs: list[SemanticBelief]) -> dict: + belief_schemas = [ + SemanticBeliefInferrer._create_belief_schema(belief) for belief in beliefs + ] + + return { + "type": "object", + "properties": dict(belief_schemas), + "required": [name for name, _ in belief_schemas], + } + + @staticmethod + def _format_message(message: ChatMessage): + return f"{message.role.upper()}:\n{message.content}" + + @staticmethod + def _format_conversation(conversation: ChatHistory): + return "\n\n".join( + [SemanticBeliefInferrer._format_message(message) for message in conversation.messages] + ) + + @staticmethod + def _format_beliefs(beliefs: list[SemanticBelief]): + return "\n".join( + [f"- {AgentSpeakGenerator.slugify(belief)}: {belief.description}" for belief in beliefs] + ) + + +class GoalAchievementInferrer(SemanticBeliefInferrer): + def __init__(self, llm: TextBeliefExtractorAgent.LLM): + super().__init__(llm) + self.goals = [] + + async def infer_from_conversation(self, conversation: ChatHistory) -> dict[str, bool]: """ - Query the LLM with the given prompt and schema, return an instance of a dict conforming - to this schema. Try ``tries`` times, or return None. + Determine which goals have been achieved based on the given conversation. - :param prompt: Prompt to be queried. - :param schema: Schema to be queried. - :return: An instance of a dict conforming to this schema, or None if failed. + :param conversation: The conversation to infer goal completion from. + :return: A mapping of goals and a boolean whether they have been achieved. """ - try_count = 0 - while try_count < tries: - try_count += 1 + if not self.goals: + return {} - try: - return await self._query_llm(prompt, schema) - except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e: - if try_count < tries: - continue - self.logger.exception( - "Failed to get LLM response after %d tries.", - try_count, - exc_info=e, - ) + goals_achieved = await asyncio.gather( + *[self._infer_goal(conversation, g) for g in self.goals] + ) + return { + f"achieved_{AgentSpeakGenerator.slugify(goal)}": achieved + for goal, achieved in zip(self.goals, goals_achieved, strict=True) + } - return None + async def _infer_goal(self, conversation: ChatHistory, goal: Goal) -> bool: + prompt = f"""{self._format_conversation(conversation)} - async def _query_llm(self, prompt: str, schema: dict) -> dict: - """ - Query an LLM with the given prompt and schema, return an instance of a dict conforming to - that schema. +Given the above conversation, what has the following goal been achieved? - :param prompt: The prompt to be queried. - :param schema: Schema to use during response. - :return: A dict conforming to this schema. - :raises httpx.HTTPStatusError: If the LLM server responded with an error. - :raises json.JSONDecodeError: If the LLM response was not valid JSON. May happen if the - response was cut off early due to length limitations. - :raises KeyError: If the LLM server responded with no error, but the response was invalid. - """ - async with httpx.AsyncClient() as client: - response = await client.post( - settings.llm_settings.local_llm_url, - json={ - "model": settings.llm_settings.local_llm_model, - "messages": [{"role": "user", "content": prompt}], - "response_format": { - "type": "json_schema", - "json_schema": { - "name": "Beliefs", - "strict": True, - "schema": schema, - }, - }, - "reasoning_effort": "low", - "temperature": self.temperature, - "stream": False, - }, - timeout=None, - ) - response.raise_for_status() +The name of the goal: {goal.name} +Description of the goal: {goal.description} - response_json = response.json() - json_message = response_json["choices"][0]["message"]["content"] - beliefs = json.loads(json_message) - return beliefs +Answer with literally only `true` or `false` (without backticks).""" + + schema = { + "type": "boolean", + } + + return await self._llm.query(prompt, schema) diff --git a/src/control_backend/core/agent_system.py b/src/control_backend/core/agent_system.py index 9d7a47f..e12a6b2 100644 --- a/src/control_backend/core/agent_system.py +++ b/src/control_backend/core/agent_system.py @@ -192,7 +192,16 @@ class BaseAgent(ABC): :param coro: The coroutine to execute as a task. """ - task = asyncio.create_task(coro) + + async def try_coro(coro_: Coroutine): + try: + await coro_ + except asyncio.CancelledError: + self.logger.debug("A behavior was canceled successfully: %s", coro_) + except Exception: + self.logger.warning("An exception occurred in a behavior.", exc_info=True) + + task = asyncio.create_task(try_coro(coro)) self._tasks.add(task) task.add_done_callback(self._tasks.discard) return task diff --git a/src/control_backend/schemas/belief_list.py b/src/control_backend/schemas/belief_list.py index ec6a7a1..b79247d 100644 --- a/src/control_backend/schemas/belief_list.py +++ b/src/control_backend/schemas/belief_list.py @@ -1,6 +1,7 @@ from pydantic import BaseModel from control_backend.schemas.program import Belief as ProgramBelief +from control_backend.schemas.program import Goal class BeliefList(BaseModel): @@ -12,3 +13,7 @@ class BeliefList(BaseModel): """ beliefs: list[ProgramBelief] + + +class GoalList(BaseModel): + goals: list[Goal] diff --git a/src/control_backend/schemas/belief_message.py b/src/control_backend/schemas/belief_message.py index 56a8a4a..51411b3 100644 --- a/src/control_backend/schemas/belief_message.py +++ b/src/control_backend/schemas/belief_message.py @@ -13,6 +13,9 @@ class Belief(BaseModel): name: str arguments: list[str] | None + # To make it hashable + model_config = {"frozen": True} + class BeliefMessage(BaseModel): """ diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index df20954..82c017e 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -117,7 +117,7 @@ class Goal(ProgramElement): :ivar can_fail: Whether we can fail to achieve the goal after executing the plan. """ - description: str + description: str = "" plan: Plan can_fail: bool = True From 3d49e44cf7c4e877e612d52919c44abf3e977706 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 7 Jan 2026 17:13:58 +0100 Subject: [PATCH 28/90] fix: complete pipeline working User interrupts still need to be tested. ref: N25B-429 --- .../agents/bdi/agentspeak_generator.py | 52 ++++- .../agents/bdi/bdi_core_agent.py | 177 ++++++++++++++++-- .../agents/bdi/bdi_program_manager.py | 74 +++++--- .../communication/ri_communication_agent.py | 3 +- src/control_backend/agents/llm/llm_agent.py | 24 ++- src/control_backend/core/agent_system.py | 1 + src/control_backend/core/config.py | 2 +- .../schemas/internal_message.py | 2 +- 8 files changed, 276 insertions(+), 59 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 1c313ce..17248a8 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -157,7 +157,7 @@ class AgentSpeakGenerator: previous_goal = None for goal in phase.goals: - self._process_goal(goal, phase, previous_goal) + self._process_goal(goal, phase, previous_goal, main_goal=True) previous_goal = goal for trigger in phase.triggers: @@ -192,6 +192,20 @@ class AgentSpeakGenerator: ] ) + # Notify outside world about transition + body.append( + AstStatement( + StatementType.DO_ACTION, + AstLiteral( + "notify_transition_phase", + [ + AstString(str(from_phase.id)), + AstString(str(to_phase.id) if to_phase else "end"), + ], + ), + ) + ) + self._asp.plans.append( AstPlan(TriggerType.ADDED_GOAL, AstLiteral("transition_phase"), context, body) ) @@ -213,6 +227,11 @@ class AgentSpeakGenerator: def _add_default_loop(self, phase: Phase) -> None: actions = [] + actions.append( + AstStatement( + StatementType.DO_ACTION, AstLiteral("notify_user_said", [AstVar("Message")]) + ) + ) actions.append(AstStatement(StatementType.REMOVE_BELIEF, AstLiteral("responded_this_turn"))) actions.append(AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("check_triggers"))) @@ -236,6 +255,7 @@ class AgentSpeakGenerator: phase: Phase, previous_goal: Goal | None = None, continues_response: bool = False, + main_goal: bool = False, ) -> None: context: list[AstExpression] = [self._astify(phase)] context.append(~self._astify(goal, achieved=True)) @@ -245,6 +265,13 @@ class AgentSpeakGenerator: context.append(~AstLiteral("responded_this_turn")) body = [] + if main_goal: # UI only needs to know about the main goals + body.append( + AstStatement( + StatementType.DO_ACTION, + AstLiteral("notify_goal_start", [AstString(self.slugify(goal))]), + ) + ) subgoals = [] for step in goal.plan.steps: @@ -283,11 +310,23 @@ class AgentSpeakGenerator: body = [] subgoals = [] + body.append( + AstStatement( + StatementType.DO_ACTION, + AstLiteral("notify_trigger_start", [AstString(self.slugify(trigger))]), + ) + ) for step in trigger.plan.steps: body.append(self._step_to_statement(step)) if isinstance(step, Goal): step.can_fail = False # triggers are continuous sequence subgoals.append(step) + body.append( + AstStatement( + StatementType.DO_ACTION, + AstLiteral("notify_trigger_end", [AstString(self.slugify(trigger))]), + ) + ) self._asp.plans.append( AstPlan( @@ -298,6 +337,9 @@ class AgentSpeakGenerator: ) ) + # Force trigger (from UI) + self._asp.plans.append(AstPlan(TriggerType.ADDED_GOAL, self._astify(trigger), [], body)) + for subgoal in subgoals: self._process_goal(subgoal, phase, continues_response=True) @@ -332,13 +374,7 @@ class AgentSpeakGenerator: @_astify.register def _(self, sb: SemanticBelief) -> AstExpression: - return AstLiteral(self.get_semantic_belief_slug(sb)) - - @staticmethod - def get_semantic_belief_slug(sb: SemanticBelief) -> str: - # If you need a method like this for other types, make a public slugify singledispatch for - # all types. - return f"semantic_{AgentSpeakGenerator._slugify_str(sb.name)}" + return AstLiteral(self.slugify(sb)) @_astify.register def _(self, ib: InferredBelief) -> AstExpression: diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 58ece29..aec8343 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -1,5 +1,6 @@ import asyncio import copy +import json import time from collections.abc import Iterable @@ -13,7 +14,7 @@ from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings from control_backend.schemas.belief_message import BeliefMessage from control_backend.schemas.llm_prompt_message import LLMPromptMessage -from control_backend.schemas.ri_message import SpeechCommand +from control_backend.schemas.ri_message import GestureCommand, RIEndpoint, SpeechCommand DELIMITER = ";\n" # TODO: temporary until we support lists in AgentSpeak @@ -155,6 +156,17 @@ class BDICoreAgent(BaseAgent): body=cmd.model_dump_json(), ) await self.send(out_msg) + case settings.agent_settings.user_interrupt_name: + content = msg.body + self.logger.debug("Received user interruption: %s", content) + + match msg.thread: + case "force_phase_transition": + self._set_goal("transition_phase") + case "force_trigger": + self._force_trigger(msg.body) + case _: + self.logger.warning("Received unknow user interruption: %s", msg) def _apply_belief_changes(self, belief_changes: BeliefMessage): """ @@ -250,6 +262,37 @@ class BDICoreAgent(BaseAgent): self.logger.debug(f"Removed {removed_count} beliefs.") + def _set_goal(self, name: str, args: Iterable[str] | None = None): + args = args or [] + + if args: + merged_args = DELIMITER.join(arg for arg in args) + new_args = (agentspeak.Literal(merged_args),) + term = agentspeak.Literal(name, new_args) + else: + term = agentspeak.Literal(name) + + self.bdi_agent.call( + agentspeak.Trigger.addition, + agentspeak.GoalType.achievement, + term, + agentspeak.runtime.Intention(), + ) + + self._wake_bdi_loop.set() + + self.logger.debug(f"Set goal !{self.format_belief_string(name, args)}.") + + def _force_trigger(self, name: str): + self.bdi_agent.call( + agentspeak.Trigger.addition, + agentspeak.GoalType.achievement, + agentspeak.Literal(name), + agentspeak.runtime.Intention(), + ) + + self.logger.info("Manually forced trigger %s.", name) + def _add_custom_actions(self) -> None: """ Add any custom actions here. Inside `@self.actions.add()`, the first argument is @@ -258,7 +301,7 @@ class BDICoreAgent(BaseAgent): """ @self.actions.add(".reply", 2) - def _reply(agent: "BDICoreAgent", term, intention): + def _reply(agent, term, intention): """ Let the LLM generate a response to a user's utterance with the current norms and goals. """ @@ -291,7 +334,7 @@ class BDICoreAgent(BaseAgent): yield @self.actions.add(".say", 1) - def _say(agent: "BDICoreAgent", term, intention): + def _say(agent, term, intention): """ Make the robot say the given text instantly. """ @@ -305,12 +348,21 @@ class BDICoreAgent(BaseAgent): sender=settings.agent_settings.bdi_core_name, body=speech_command.model_dump_json(), ) - # TODO: add to conversation history + self.add_behavior(self.send(speech_message)) + + chat_history_message = InternalMessage( + to=settings.agent_settings.llm_name, + thread="assistant_message", + body=str(message_text), + ) + + self.add_behavior(self.send(chat_history_message)) + yield @self.actions.add(".gesture", 2) - def _gesture(agent: "BDICoreAgent", term, intention): + def _gesture(agent, term, intention): """ Make the robot perform the given gesture instantly. """ @@ -323,13 +375,113 @@ class BDICoreAgent(BaseAgent): gesture_name, ) - # gesture = Gesture(type=gesture_type, name=gesture_name) - # gesture_message = InternalMessage( - # to=settings.agent_settings.robot_gesture_name, - # sender=settings.agent_settings.bdi_core_name, - # body=gesture.model_dump_json(), - # ) - # asyncio.create_task(agent.send(gesture_message)) + if str(gesture_type) == "single": + endpoint = RIEndpoint.GESTURE_SINGLE + elif str(gesture_type) == "tag": + endpoint = RIEndpoint.GESTURE_TAG + else: + self.logger.warning("Gesture type %s could not be resolved.", gesture_type) + endpoint = RIEndpoint.GESTURE_SINGLE + + gesture_command = GestureCommand(endpoint=endpoint, data=gesture_name) + gesture_message = InternalMessage( + to=settings.agent_settings.robot_gesture_name, + sender=settings.agent_settings.bdi_core_name, + body=gesture_command.model_dump_json(), + ) + self.add_behavior(self.send(gesture_message)) + yield + + @self.actions.add(".notify_user_said", 1) + def _notify_user_said(agent, term, intention): + user_said = agentspeak.grounded(term.args[0], intention.scope) + + msg = InternalMessage( + to=settings.agent_settings.llm_name, thread="user_message", body=str(user_said) + ) + + self.add_behavior(self.send(msg)) + + yield + + @self.actions.add(".notify_trigger_start", 1) + def _notify_trigger_start(agent, term, intention): + """ + Notify the UI about the trigger we just started doing. + """ + trigger_name = agentspeak.grounded(term.args[0], intention.scope) + + self.logger.debug("Started trigger %s", trigger_name) + + msg = InternalMessage( + to=settings.agent_settings.user_interrupt_name, + sender=self.name, + thread="trigger_start", + body=str(trigger_name), + ) + + # TODO: check with Pim + self.add_behavior(self.send(msg)) + + yield + + @self.actions.add(".notify_trigger_end", 1) + def _notify_trigger_end(agent, term, intention): + """ + Notify the UI about the trigger we just started doing. + """ + trigger_name = agentspeak.grounded(term.args[0], intention.scope) + + self.logger.debug("Finished trigger %s", trigger_name) + + msg = InternalMessage( + to=settings.agent_settings.user_interrupt_name, + sender=self.name, + thread="trigger_end", + body=str(trigger_name), + ) + + # TODO: check with Pim + self.add_behavior(self.send(msg)) + + yield + + @self.actions.add(".notify_goal_start", 1) + def _notify_goal_start(agent, term, intention): + """ + Notify the UI about the goal we just started chasing. + """ + goal_name = agentspeak.grounded(term.args[0], intention.scope) + + self.logger.debug("Started chasing goal %s", goal_name) + + msg = InternalMessage( + to=settings.agent_settings.user_interrupt_name, + sender=self.name, + thread="goal_start", + body=str(goal_name), + ) + + self.add_behavior(self.send(msg)) + + yield + + @self.actions.add(".notify_transition_phase", 2) + def _notify_transition_phase(agent, term, intention): + """ + Notify the BDI program manager about a phase transition. + """ + old = agentspeak.grounded(term.args[0], intention.scope) + new = agentspeak.grounded(term.args[1], intention.scope) + + msg = InternalMessage( + to=settings.agent_settings.bdi_program_manager_name, + thread="transition_phase", + body=json.dumps({"old": str(old), "new": str(new)}), + ) + + self.add_behavior(self.send(msg)) + yield async def _send_to_llm(self, text: str, norms: str, goals: str): @@ -341,6 +493,7 @@ class BDICoreAgent(BaseAgent): to=settings.agent_settings.llm_name, sender=self.name, body=prompt.model_dump_json(), + thread="prompt_message", ) await self.send(msg) self.logger.info("Message sent to LLM agent: %s", text) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 54e7196..ba000de 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -1,4 +1,5 @@ import asyncio +import json import zmq from pydantic import ValidationError @@ -9,7 +10,7 @@ from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.config import settings from control_backend.schemas.belief_list import BeliefList from control_backend.schemas.internal_message import InternalMessage -from control_backend.schemas.program import Belief, ConditionalNorm, InferredBelief, Program +from control_backend.schemas.program import Belief, ConditionalNorm, InferredBelief, Phase, Program class BDIProgramManager(BaseAgent): @@ -24,20 +25,20 @@ class BDIProgramManager(BaseAgent): :ivar sub_socket: The ZMQ SUB socket used to receive program updates. """ + _program: Program + _phase: Phase | None + def __init__(self, **kwargs): super().__init__(**kwargs) self.sub_socket = None + def _initialize_internal_state(self, program: Program): + self._program = program + self._phase = program.phases[0] # start in first phase + async def _create_agentspeak_and_send_to_bdi(self, program: Program): """ - Convert a received program into BDI beliefs and send them to the BDI Core Agent. - - Currently, it takes the **first phase** of the program and extracts: - - **Norms**: Constraints or rules the agent must follow. - - **Goals**: Objectives the agent must achieve. - - These are sent as a ``BeliefMessage`` with ``replace=True``, meaning they will - overwrite any existing norms/goals of the same name in the BDI agent. + Convert a received program into an AgentSpeak file and send it to the BDI Core Agent. :param program: The program object received from the API. """ @@ -59,17 +60,44 @@ class BDIProgramManager(BaseAgent): await self.send(msg) - @staticmethod - def _extract_beliefs_from_program(program: Program) -> list[Belief]: + def handle_message(self, msg: InternalMessage): + match msg.thread: + case "transition_phase": + phases = json.loads(msg.body) + + self._transition_phase(phases["old"], phases["new"]) + + def _transition_phase(self, old: str, new: str): + assert old == str(self._phase.id) + + if new == "end": + self._phase = None + return + + for phase in self._program.phases: + if str(phase.id) == new: + self._phase = phase + + self._send_beliefs_to_semantic_belief_extractor() + + # Notify user interaction agent + msg = InternalMessage( + to=settings.agent_settings.user_interrupt_name, + thread="transition_phase", + body=str(self._phase.id), + ) + + self.add_behavior(self.send(msg)) + + def _extract_current_beliefs(self) -> list[Belief]: beliefs: list[Belief] = [] - for phase in program.phases: - for norm in phase.norms: - if isinstance(norm, ConditionalNorm): - beliefs += BDIProgramManager._extract_beliefs_from_belief(norm.condition) + for norm in self._phase.norms: + if isinstance(norm, ConditionalNorm): + beliefs += self._extract_beliefs_from_belief(norm.condition) - for trigger in phase.triggers: - beliefs += BDIProgramManager._extract_beliefs_from_belief(trigger.condition) + for trigger in self._phase.triggers: + beliefs += self._extract_beliefs_from_belief(trigger.condition) return beliefs @@ -81,13 +109,11 @@ class BDIProgramManager(BaseAgent): ) + BDIProgramManager._extract_beliefs_from_belief(belief.right) return [belief] - async def _send_beliefs_to_semantic_belief_extractor(self, program: Program): + async def _send_beliefs_to_semantic_belief_extractor(self): """ Extract beliefs from the program and send them to the Semantic Belief Extractor Agent. - - :param program: The program received from the API. """ - beliefs = BeliefList(beliefs=self._extract_beliefs_from_program(program)) + beliefs = BeliefList(beliefs=self._extract_current_beliefs()) message = InternalMessage( to=settings.agent_settings.text_belief_extractor_name, @@ -111,12 +137,14 @@ class BDIProgramManager(BaseAgent): try: program = Program.model_validate_json(body) except ValidationError: - self.logger.exception("Received an invalid program.") + self.logger.warning("Received an invalid program.") continue + self._initialize_internal_state(program) + await asyncio.gather( self._create_agentspeak_and_send_to_bdi(program), - self._send_beliefs_to_semantic_belief_extractor(program), + self._send_beliefs_to_semantic_belief_extractor(), ) async def setup(self): diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 34e5b25..34d3a5a 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -248,7 +248,8 @@ class RICommunicationAgent(BaseAgent): self._req_socket.recv_json(), timeout=seconds_to_wait_total / 2 ) - self.logger.debug(f'Received message "{message}" from RI.') + if message["endpoint"] and message["endpoint"] != "ping": + self.logger.debug(f'Received message "{message}" from RI.') if "endpoint" not in message: self.logger.warning("No received endpoint in message, expected ping endpoint.") continue diff --git a/src/control_backend/agents/llm/llm_agent.py b/src/control_backend/agents/llm/llm_agent.py index 17edec9..3e19c49 100644 --- a/src/control_backend/agents/llm/llm_agent.py +++ b/src/control_backend/agents/llm/llm_agent.py @@ -46,12 +46,17 @@ class LLMAgent(BaseAgent): :param msg: The received internal message. """ if msg.sender == settings.agent_settings.bdi_core_name: - self.logger.debug("Processing message from BDI core.") - try: - prompt_message = LLMPromptMessage.model_validate_json(msg.body) - await self._process_bdi_message(prompt_message) - except ValidationError: - self.logger.debug("Prompt message from BDI core is invalid.") + match msg.thread: + case "prompt_message": + try: + prompt_message = LLMPromptMessage.model_validate_json(msg.body) + await self._process_bdi_message(prompt_message) + except ValidationError: + self.logger.debug("Prompt message from BDI core is invalid.") + case "assistant_message": + self.history.append({"role": "assistant", "content": msg.body}) + case "user_message": + self.history.append({"role": "user", "content": msg.body}) else: self.logger.debug("Message ignored (not from BDI core.") @@ -114,13 +119,6 @@ class LLMAgent(BaseAgent): :param goals: Goals the LLM should achieve. :yield: Fragments of the LLM-generated content (e.g., sentences/phrases). """ - self.history.append( - { - "role": "user", - "content": prompt, - } - ) - instructions = LLMInstructions(norms if norms else None, goals if goals else None) messages = [ { diff --git a/src/control_backend/core/agent_system.py b/src/control_backend/core/agent_system.py index 9d7a47f..fc418bb 100644 --- a/src/control_backend/core/agent_system.py +++ b/src/control_backend/core/agent_system.py @@ -131,6 +131,7 @@ class BaseAgent(ABC): :param message: The message to send. """ target = AgentDirectory.get(message.to) + message.sender = self.name if target: await target.inbox.put(message) self.logger.debug(f"Sent message {message.body} to {message.to} via regular inbox.") diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index 8a7267c..353a408 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -75,7 +75,7 @@ class BehaviourSettings(BaseModel): # VAD settings vad_prob_threshold: float = 0.5 vad_initial_since_speech: int = 100 - vad_non_speech_patience_chunks: int = 3 + vad_non_speech_patience_chunks: int = 15 # transcription behaviour transcription_max_concurrent_tasks: int = 3 diff --git a/src/control_backend/schemas/internal_message.py b/src/control_backend/schemas/internal_message.py index 071d884..14278c0 100644 --- a/src/control_backend/schemas/internal_message.py +++ b/src/control_backend/schemas/internal_message.py @@ -12,6 +12,6 @@ class InternalMessage(BaseModel): """ to: str - sender: str + sender: str | None = None body: str thread: str | None = None From 8a77e8e1c756dfc9347e09cd7642a82216401410 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 7 Jan 2026 17:31:24 +0100 Subject: [PATCH 29/90] feat: check goals only for this phase Since conversation history still remains we can still check at a later point. ref: N25B-429 --- .../agents/bdi/bdi_program_manager.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index fefd6a7..12d8c6a 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -86,6 +86,7 @@ class BDIProgramManager(BaseAgent): self._phase = phase self._send_beliefs_to_semantic_belief_extractor() + self._send_goals_to_semantic_belief_extractor() # Notify user interaction agent msg = InternalMessage( @@ -131,12 +132,10 @@ class BDIProgramManager(BaseAgent): await self.send(message) - @staticmethod - def _extract_goals_from_program(program: Program) -> list[Goal]: + def _extract_current_goals(self) -> list[Goal]: """ Extract all goals from the program, including subgoals. - :param program: The program received from the API. :return: A list of Goal objects. """ goals: list[Goal] = [] @@ -148,19 +147,16 @@ class BDIProgramManager(BaseAgent): goals_.extend(extract_goals_from_goal(plan)) return goals_ - for phase in program.phases: - for goal in phase.goals: - goals.extend(extract_goals_from_goal(goal)) + for goal in self._phase.goals: + goals.extend(extract_goals_from_goal(goal)) return goals - async def _send_goals_to_semantic_belief_extractor(self, program: Program): + async def _send_goals_to_semantic_belief_extractor(self): """ - Extract goals from the program and send them to the Semantic Belief Extractor Agent. - - :param program: The program received from the API. + Extract goals for the current phase and send them to the Semantic Belief Extractor Agent. """ - goals = GoalList(goals=self._extract_goals_from_program(program)) + goals = GoalList(goals=self._extract_current_goals()) message = InternalMessage( to=settings.agent_settings.text_belief_extractor_name, @@ -192,7 +188,7 @@ class BDIProgramManager(BaseAgent): await asyncio.gather( self._create_agentspeak_and_send_to_bdi(program), self._send_beliefs_to_semantic_belief_extractor(), - self._send_goals_to_semantic_belief_extractor(program), + self._send_goals_to_semantic_belief_extractor(), ) async def setup(self): From be6bbbb849a8dd4dd6ca02af7c298b38183d9b21 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Wed, 7 Jan 2026 17:42:54 +0100 Subject: [PATCH 30/90] feat: added endpoint userinterrupt to userinterrupt ref: N25B-400 --- src/control_backend/agents/bdi/agentspeak.asl | 45 +++++++++++++ .../user_interrupt/user_interrupt_agent.py | 63 ++++++++++++----- .../api/v1/endpoints/button_pressed.py | 31 --------- .../api/v1/endpoints/user_interact.py | 67 +++++++++++++++++++ src/control_backend/api/v1/router.py | 4 +- 5 files changed, 162 insertions(+), 48 deletions(-) create mode 100644 src/control_backend/agents/bdi/agentspeak.asl delete mode 100644 src/control_backend/api/v1/endpoints/button_pressed.py create mode 100644 src/control_backend/api/v1/endpoints/user_interact.py diff --git a/src/control_backend/agents/bdi/agentspeak.asl b/src/control_backend/agents/bdi/agentspeak.asl new file mode 100644 index 0000000..7f71fbd --- /dev/null +++ b/src/control_backend/agents/bdi/agentspeak.asl @@ -0,0 +1,45 @@ +phase("9922935f-ec70-4792-9a61-37a129e1ec14"). +keyword_said(Keyword) :- (user_said(Message) & .substring(Keyword, Message, Pos)) & (Pos >= 0). + + ++!reply_with_goal(Goal) + : user_said(Message) + <- +responded_this_turn; + .findall(Norm, norm(Norm), Norms); + .reply_with_goal(Message, Norms, Goal). + ++!say(Text) + <- +responded_this_turn; + .say(Text). + ++!reply + : user_said(Message) + <- +responded_this_turn; + .findall(Norm, norm(Norm), Norms); + .reply(Message, Norms). + ++user_said(Message) + : phase("9922935f-ec70-4792-9a61-37a129e1ec14") + <- .notify_user_said(Message); + -responded_this_turn; + !check_triggers; + !transition_phase. + ++!transition_phase + : phase("9922935f-ec70-4792-9a61-37a129e1ec14") & + not responded_this_turn + <- -phase("9922935f-ec70-4792-9a61-37a129e1ec14"); + +phase("end"); + ?user_said(Message); + -+user_said(Message); + .notify_transition_phase("9922935f-ec70-4792-9a61-37a129e1ec14", "end"). + ++user_said(Message) + : phase("end") + <- !reply. + ++!check_triggers + <- true. + ++!transition_phase + <- true. diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index b2efc41..af00a7b 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -1,3 +1,4 @@ +import asyncio import json import zmq @@ -30,6 +31,26 @@ class UserInterruptAgent(BaseAgent): def __init__(self, **kwargs): super().__init__(**kwargs) self.sub_socket = None + self.pub_socket = None + + async def setup(self): + """ + Initialize the agent. + + Connects the internal ZMQ SUB socket and subscribes to the 'button_pressed' topic. + Starts the background behavior to receive the user interrupts. + """ + context = Context.instance() + + self.sub_socket = context.socket(zmq.SUB) + self.sub_socket.connect(settings.zmq_settings.internal_sub_address) + self.sub_socket.subscribe("button_pressed") + + self.pub_socket = context.socket(zmq.PUB) + self.pub_socket.connect(settings.zmq_settings.internal_pub_address) + + self.add_behavior(self._receive_button_event()) + self.add_behavior(self.test_sending_behaviour()) async def _receive_button_event(self): """ @@ -78,6 +99,33 @@ class UserInterruptAgent(BaseAgent): event_context, ) + async def test_sending_behaviour(self): + self.logger.info("Starting simple test sending behaviour...") + + while True: + try: + test_data = {"type": "heartbeat", "status": "ok"} + + await self._send_experiment_update(test_data) + + except zmq.ZMQError as ze: + self.logger.error(f"ZMQ error: {ze}") + except Exception as e: + self.logger.error(f"Error: {e}") + + await asyncio.sleep(2) + + async def _send_experiment_update(self, data): + """ + Sends an update to the 'experiment' topic. + The SSE endpoint will pick this up and push it to the UI. + """ + if self.pub_socket: + topic = b"experiment" + body = json.dumps(data).encode("utf-8") + await self.pub_socket.send_multipart([topic, body]) + self.logger.debug(f"Sent experiment update: {data}") + async def _send_to_speech_agent(self, text_to_say: str): """ method to send prioritized speech command to RobotSpeechAgent. @@ -129,18 +177,3 @@ class UserInterruptAgent(BaseAgent): "Sent button_override belief with id '%s' to Program manager.", belief_id, ) - - async def setup(self): - """ - Initialize the agent. - - Connects the internal ZMQ SUB socket and subscribes to the 'button_pressed' topic. - Starts the background behavior to receive the user interrupts. - """ - context = Context.instance() - - self.sub_socket = context.socket(zmq.SUB) - self.sub_socket.connect(settings.zmq_settings.internal_sub_address) - self.sub_socket.subscribe("button_pressed") - - self.add_behavior(self._receive_button_event()) diff --git a/src/control_backend/api/v1/endpoints/button_pressed.py b/src/control_backend/api/v1/endpoints/button_pressed.py deleted file mode 100644 index 5a94a53..0000000 --- a/src/control_backend/api/v1/endpoints/button_pressed.py +++ /dev/null @@ -1,31 +0,0 @@ -import logging - -from fastapi import APIRouter, Request - -from control_backend.schemas.events import ButtonPressedEvent - -logger = logging.getLogger(__name__) -router = APIRouter() - - -@router.post("/button_pressed", status_code=202) -async def receive_button_event(event: ButtonPressedEvent, request: Request): - """ - Endpoint to handle external button press events. - - Validates the event payload and publishes it to the internal 'button_pressed' topic. - Subscribers (in this case user_interrupt_agent) will pick this up to trigger - specific behaviors or state changes. - - :param event: The parsed ButtonPressedEvent object. - :param request: The FastAPI request object. - """ - logger.debug("Received button event: %s | %s", event.type, event.context) - - topic = b"button_pressed" - body = event.model_dump_json().encode() - - pub_socket = request.app.state.endpoints_pub_socket - await pub_socket.send_multipart([topic, body]) - - return {"status": "Event received"} diff --git a/src/control_backend/api/v1/endpoints/user_interact.py b/src/control_backend/api/v1/endpoints/user_interact.py new file mode 100644 index 0000000..3d3406e --- /dev/null +++ b/src/control_backend/api/v1/endpoints/user_interact.py @@ -0,0 +1,67 @@ +import asyncio +import logging + +import zmq +import zmq.asyncio +from fastapi import APIRouter, Request +from fastapi.responses import StreamingResponse +from zmq.asyncio import Context + +from control_backend.core.config import settings +from control_backend.schemas.events import ButtonPressedEvent + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.post("/button_pressed", status_code=202) +async def receive_button_event(event: ButtonPressedEvent, request: Request): + """ + Endpoint to handle external button press events. + + Validates the event payload and publishes it to the internal 'button_pressed' topic. + Subscribers (in this case user_interrupt_agent) will pick this up to trigger + specific behaviors or state changes. + + :param event: The parsed ButtonPressedEvent object. + :param request: The FastAPI request object. + """ + logger.debug("Received button event: %s | %s", event.type, event.context) + + topic = b"button_pressed" + body = event.model_dump_json().encode() + + pub_socket = request.app.state.endpoints_pub_socket + await pub_socket.send_multipart([topic, body]) + + return {"status": "Event received"} + + +@router.get("/experiment_stream") +async def experiment_stream(request: Request): + # Use the asyncio-compatible context + context = Context.instance() + socket = context.socket(zmq.SUB) + + # Connect and subscribe + socket.connect(settings.zmq_settings.internal_sub_address) + socket.subscribe(b"experiment") + + async def gen(): + try: + while True: + # Check if client closed the tab + if await request.is_disconnected(): + logger.info("Client disconnected from experiment stream.") + break + + try: + parts = await asyncio.wait_for(socket.recv_multipart(), timeout=1.0) + _, message = parts + yield f"data: {message.decode().strip()}\n\n" + except TimeoutError: + continue + finally: + socket.close() + + return StreamingResponse(gen(), media_type="text/event-stream") diff --git a/src/control_backend/api/v1/router.py b/src/control_backend/api/v1/router.py index ebba0db..c130ad3 100644 --- a/src/control_backend/api/v1/router.py +++ b/src/control_backend/api/v1/router.py @@ -1,6 +1,6 @@ from fastapi.routing import APIRouter -from control_backend.api.v1.endpoints import button_pressed, logs, message, program, robot, sse +from control_backend.api.v1.endpoints import logs, message, program, robot, sse, user_interact api_router = APIRouter() @@ -14,4 +14,4 @@ api_router.include_router(logs.router, tags=["Logs"]) api_router.include_router(program.router, tags=["Program"]) -api_router.include_router(button_pressed.router, tags=["Button Pressed Events"]) +api_router.include_router(user_interact.router, tags=["Button Pressed Events"]) From 93d67ccb66ace8386b5ef9286af4761d31bd5344 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Wed, 7 Jan 2026 17:50:47 +0100 Subject: [PATCH 31/90] feat: add reset functionality to semantic belief extractor ref: N25B-432 --- .../agents/bdi/text_belief_extractor_agent.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index 7e3570f..feabf40 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -116,9 +116,19 @@ class TextBeliefExtractorAgent(BaseAgent): self._handle_beliefs_message(msg) case "goals": self._handle_goals_message(msg) + case "conversation_history": + if msg.body == "reset": + self._reset() case _: self.logger.warning("Received unexpected message from %s", msg.sender) + def _reset(self): + self.conversation = ChatHistory(messages=[]) + self.belief_inferrer.available_beliefs.clear() + self._current_beliefs = BeliefState() + self.goal_inferrer.goals.clear() + self._current_goal_completions = {} + def _handle_beliefs_message(self, msg: InternalMessage): try: belief_list = BeliefList.model_validate_json(msg.body) From 5a61225c6f40d45718723a7bf559e607974361a3 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 7 Jan 2026 18:10:13 +0100 Subject: [PATCH 32/90] feat: reset extractor history ref: N25B-429 --- .../agents/bdi/bdi_program_manager.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 1ee3e3c..8b8c68f 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -175,13 +175,19 @@ class BDIProgramManager(BaseAgent): """ message = InternalMessage( to=settings.agent_settings.llm_name, - sender=self.name, body="clear_history", - threads="clear history message", ) await self.send(message) self.logger.debug("Sent message to LLM agent to clear history.") + extractor_msg = InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + thread="conversation_history", + body="reset", + ) + await self.send(extractor_msg) + self.logger.debug("Sent message to extractor agent to clear history.") + async def _receive_programs(self): """ Continuous loop that receives program updates from the HTTP endpoint. @@ -201,11 +207,12 @@ class BDIProgramManager(BaseAgent): self._initialize_internal_state(program) + await self._send_clear_llm_history() + await asyncio.gather( self._create_agentspeak_and_send_to_bdi(program), self._send_beliefs_to_semantic_belief_extractor(), self._send_goals_to_semantic_belief_extractor(), - self._send_clear_llm_history(), ) async def setup(self): From be88323cf76333f0b2dfcb98c2d93b1723977a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Otgaar?= Date: Wed, 7 Jan 2026 18:24:35 +0100 Subject: [PATCH 33/90] chore: add one endpoint fo avoid errors --- .../agents/user_interrupt/user_interrupt_agent.py | 8 ++++---- src/control_backend/schemas/ri_message.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 842231a..50df979 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -82,6 +82,7 @@ class UserInterruptAgent(BaseAgent): self.logger.info("Sent pause command.") else: self.logger.info("Sent resume command.") + elif event_type in ["next_phase", "reset_phase", "reset_experiment"]: await self._send_experiment_control_to_bdi_core(event_type) @@ -173,11 +174,11 @@ class UserInterruptAgent(BaseAgent): "Sent button_override belief with id '%s' to Program manager.", belief_id, ) - - async def _send_pause_command(self, pause : bool): + + async def _send_pause_command(self, pause: bool): """ Send a pause command to the Robot Interface via the RI Communication Agent. - Send a pause command to the other internal agents; for now just VAD agent. + Send a pause command to the other internal agents; for now just VAD agent. """ cmd = PauseCommand(data=pause) message = InternalMessage( @@ -206,7 +207,6 @@ class UserInterruptAgent(BaseAgent): await self.send(vad_message) self.logger.info("Sent resume command to VAD Agent and RI Communication Agent.") - async def setup(self): """ Initialize the agent. diff --git a/src/control_backend/schemas/ri_message.py b/src/control_backend/schemas/ri_message.py index 7c1ef22..e6eafa3 100644 --- a/src/control_backend/schemas/ri_message.py +++ b/src/control_backend/schemas/ri_message.py @@ -14,6 +14,7 @@ class RIEndpoint(str, Enum): GESTURE_TAG = "actuate/gesture/tag" PING = "ping" NEGOTIATE_PORTS = "negotiate/ports" + PAUSE = "" class RIMessage(BaseModel): @@ -65,6 +66,7 @@ class GestureCommand(RIMessage): raise ValueError("endpoint must be GESTURE_SINGLE or GESTURE_TAG") return self + class PauseCommand(RIMessage): """ A specific command to pause or unpause the robot's actions. From 365d449666e30ce7def496e394998bab95e5de56 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Wed, 7 Jan 2026 22:41:59 +0100 Subject: [PATCH 34/90] feat: commit before I can merge new changes ref: N25B-400 --- src/control_backend/agents/bdi/agentspeak.asl | 26 +++++++++++++++---- .../agents/bdi/bdi_program_manager.py | 1 + .../user_interrupt/user_interrupt_agent.py | 19 +++++++++++++- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak.asl b/src/control_backend/agents/bdi/agentspeak.asl index 7f71fbd..e6e1fb0 100644 --- a/src/control_backend/agents/bdi/agentspeak.asl +++ b/src/control_backend/agents/bdi/agentspeak.asl @@ -1,4 +1,4 @@ -phase("9922935f-ec70-4792-9a61-37a129e1ec14"). +phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49"). keyword_said(Keyword) :- (user_said(Message) & .substring(Keyword, Message, Pos)) & (Pos >= 0). @@ -19,20 +19,36 @@ keyword_said(Keyword) :- (user_said(Message) & .substring(Keyword, Message, Pos) .reply(Message, Norms). +user_said(Message) - : phase("9922935f-ec70-4792-9a61-37a129e1ec14") + : phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49") <- .notify_user_said(Message); -responded_this_turn; !check_triggers; !transition_phase. +!transition_phase - : phase("9922935f-ec70-4792-9a61-37a129e1ec14") & + : phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49") & not responded_this_turn - <- -phase("9922935f-ec70-4792-9a61-37a129e1ec14"); + <- -phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49"); + +phase("1fc60869-86db-483d-b475-b8ecdec4bba8"); + ?user_said(Message); + -+user_said(Message); + .notify_transition_phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49", "1fc60869-86db-483d-b475-b8ecdec4bba8"). + ++user_said(Message) + : phase("1fc60869-86db-483d-b475-b8ecdec4bba8") + <- .notify_user_said(Message); + -responded_this_turn; + !check_triggers; + !transition_phase. + ++!transition_phase + : phase("1fc60869-86db-483d-b475-b8ecdec4bba8") & + not responded_this_turn + <- -phase("1fc60869-86db-483d-b475-b8ecdec4bba8"); +phase("end"); ?user_said(Message); -+user_said(Message); - .notify_transition_phase("9922935f-ec70-4792-9a61-37a129e1ec14", "end"). + .notify_transition_phase("1fc60869-86db-483d-b475-b8ecdec4bba8", "end"). +user_said(Message) : phase("end") diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index fefd6a7..29ff859 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -93,6 +93,7 @@ class BDIProgramManager(BaseAgent): thread="transition_phase", body=str(self._phase.id), ) + self.logger.info(f"Transitioned to phase {new}, notifying UserInterruptAgent.") self.add_behavior(self.send(msg)) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index af00a7b..cfb6d2f 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -50,7 +50,7 @@ class UserInterruptAgent(BaseAgent): self.pub_socket.connect(settings.zmq_settings.internal_pub_address) self.add_behavior(self._receive_button_event()) - self.add_behavior(self.test_sending_behaviour()) + # self.add_behavior(self.test_sending_behaviour()) async def _receive_button_event(self): """ @@ -99,6 +99,23 @@ class UserInterruptAgent(BaseAgent): event_context, ) + async def handle_message(self, msg: InternalMessage): + """ + Handle commands received from other internal Python agents. + """ + match msg.thread: + case "transition_phase": + new_phase_id = msg.body + self.logger.info(f"Phase transition detected: {new_phase_id}") + + payload = {"type": "phase_update", "phase_id": new_phase_id} + + await self._send_experiment_update(payload) + + case _: + self.logger.debug(f"Received internal message on unhandled thread: {msg.thread}") + + # moet weg!!!!! async def test_sending_behaviour(self): self.logger.info("Starting simple test sending behaviour...") From 4bf2be63599998ef8d198abb5854d30ff708d2e7 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Thu, 8 Jan 2026 09:56:10 +0100 Subject: [PATCH 35/90] feat: added a functionality for monitoring page ref: N25B-400 --- src/control_backend/agents/bdi/agentspeak.asl | 35 +++--- .../agents/bdi/agentspeak_generator.py | 24 ++-- .../agents/bdi/bdi_core_agent.py | 9 ++ .../agents/bdi/bdi_program_manager.py | 23 +++- .../user_interrupt/user_interrupt_agent.py | 110 +++++++++++++++++- 5 files changed, 161 insertions(+), 40 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak.asl b/src/control_backend/agents/bdi/agentspeak.asl index e6e1fb0..399566c 100644 --- a/src/control_backend/agents/bdi/agentspeak.asl +++ b/src/control_backend/agents/bdi/agentspeak.asl @@ -1,5 +1,6 @@ -phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49"). +phase("db4c68c3-0316-4905-a8db-22dd5bec7abf"). keyword_said(Keyword) :- (user_said(Message) & .substring(Keyword, Message, Pos)) & (Pos >= 0). +norm("do nothing and make a little dance, do a little laugh") :- phase("db4c68c3-0316-4905-a8db-22dd5bec7abf") & keyword_said("hi"). +!reply_with_goal(Goal) @@ -19,36 +20,30 @@ keyword_said(Keyword) :- (user_said(Message) & .substring(Keyword, Message, Pos) .reply(Message, Norms). +user_said(Message) - : phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49") + : phase("db4c68c3-0316-4905-a8db-22dd5bec7abf") <- .notify_user_said(Message); -responded_this_turn; !check_triggers; !transition_phase. -+!transition_phase - : phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49") & - not responded_this_turn - <- -phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49"); - +phase("1fc60869-86db-483d-b475-b8ecdec4bba8"); - ?user_said(Message); - -+user_said(Message); - .notify_transition_phase("0e0f239c-efe9-442c-bdd7-3aabfccd1c49", "1fc60869-86db-483d-b475-b8ecdec4bba8"). ++!check_triggers + : phase("db4c68c3-0316-4905-a8db-22dd5bec7abf") & + semantic_hello + <- .notify_trigger_start("trigger_"); + .notify_trigger_end("trigger_"). -+user_said(Message) - : phase("1fc60869-86db-483d-b475-b8ecdec4bba8") - <- .notify_user_said(Message); - -responded_this_turn; - !check_triggers; - !transition_phase. ++!trigger_ + <- .notify_trigger_start("trigger_"); + .notify_trigger_end("trigger_"). +!transition_phase - : phase("1fc60869-86db-483d-b475-b8ecdec4bba8") & + : phase("db4c68c3-0316-4905-a8db-22dd5bec7abf") & not responded_this_turn - <- -phase("1fc60869-86db-483d-b475-b8ecdec4bba8"); + <- .notify_transition_phase("db4c68c3-0316-4905-a8db-22dd5bec7abf", "end"); + -phase("db4c68c3-0316-4905-a8db-22dd5bec7abf"); +phase("end"); ?user_said(Message); - -+user_said(Message); - .notify_transition_phase("1fc60869-86db-483d-b475-b8ecdec4bba8", "end"). + -+user_said(Message). +user_said(Message) : phase("end") diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 17248a8..18cb794 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -176,6 +176,16 @@ class AgentSpeakGenerator: context.append(self._astify(from_phase.goals[-1], achieved=True)) body = [ + AstStatement( + StatementType.DO_ACTION, + AstLiteral( + "notify_transition_phase", + [ + AstString(str(from_phase.id)), + AstString(str(to_phase.id) if to_phase else "end"), + ], + ), + ), AstStatement(StatementType.REMOVE_BELIEF, from_phase_ast), AstStatement(StatementType.ADD_BELIEF, to_phase_ast), ] @@ -192,20 +202,6 @@ class AgentSpeakGenerator: ] ) - # Notify outside world about transition - body.append( - AstStatement( - StatementType.DO_ACTION, - AstLiteral( - "notify_transition_phase", - [ - AstString(str(from_phase.id)), - AstString(str(to_phase.id) if to_phase else "end"), - ], - ), - ) - ) - self._asp.plans.append( AstPlan(TriggerType.ADDED_GOAL, AstLiteral("transition_phase"), context, body) ) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 99bea80..94232e4 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -311,6 +311,15 @@ class BDICoreAgent(BaseAgent): message_text = agentspeak.grounded(term.args[0], intention.scope) norms = agentspeak.grounded(term.args[1], intention.scope) + norm_update_message = InternalMessage( + to=settings.agent_settings.user_interrupt_name, + sender=self.name, + thread="active_norms_update", + body=str(norms), + ) + + self.add_behavior(self.send(norm_update_message)) + self.logger.debug("Norms: %s", norms) self.logger.debug("User text: %s", message_text) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index ba022eb..7899e3c 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -75,7 +75,12 @@ class BDIProgramManager(BaseAgent): self._transition_phase(phases["old"], phases["new"]) def _transition_phase(self, old: str, new: str): - assert old == str(self._phase.id) + if old != str(self._phase.id): + self.logger.warning( + f"Phase transition desync detected! ASL requested move from '{old}', " + f"but Python is currently in '{self._phase.id}'. Request ignored." + ) + return if new == "end": self._phase = None @@ -208,6 +213,7 @@ class BDIProgramManager(BaseAgent): self._initialize_internal_state(program) + await self._send_program_to_user_interrupt(program) await self._send_clear_llm_history() await asyncio.gather( @@ -216,6 +222,21 @@ class BDIProgramManager(BaseAgent): self._send_goals_to_semantic_belief_extractor(), ) + async def _send_program_to_user_interrupt(self, program: Program): + """ + Send the received program to the User Interrupt Agent. + + :param program: The program object received from the API. + """ + msg = InternalMessage( + sender=self.name, + to=settings.agent_settings.user_interrupt_name, + body=program.model_dump_json(), + thread="new_program", + ) + + await self.send(msg) + async def setup(self): """ Initialize the agent. diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index cfb6d2f..b762e68 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -5,8 +5,10 @@ import zmq from zmq.asyncio import Context from control_backend.agents import BaseAgent +from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings +from control_backend.schemas.program import Program from control_backend.schemas.ri_message import GestureCommand, RIEndpoint, SpeechCommand @@ -32,6 +34,16 @@ class UserInterruptAgent(BaseAgent): super().__init__(**kwargs) self.sub_socket = None self.pub_socket = None + self._trigger_map = {} + self._trigger_reverse_map = {} + + self._goal_map = {} + self._goal_reverse_map = {} + + self._cond_norm_map = {} + self._cond_norm_reverse_map = {} + + self._belief_condition_map = {} async def setup(self): """ @@ -87,11 +99,19 @@ class UserInterruptAgent(BaseAgent): event_context, ) elif event_type == "override": - await self._send_to_program_manager(event_context) - self.logger.info( - "Forwarded button press (override) with context '%s' to BDIProgramManager.", - event_context, - ) + ui_id = str(event_context) + if asl_trigger := self._trigger_map.get(ui_id): + await self._send_to_bdi("force_trigger", asl_trigger) + self.logger.info( + "Forwarded button press (override) with context '%s' to BDI Core.", + event_context, + ) + else: + await self._send_to_program_manager(event_context) + self.logger.info( + "Forwarded button press (override) with context '%s' to BDIProgramManager.", + event_context, + ) else: self.logger.warning( "Received button press with unknown type '%s' (context: '%s').", @@ -104,6 +124,26 @@ class UserInterruptAgent(BaseAgent): Handle commands received from other internal Python agents. """ match msg.thread: + case "new_program": + self._create_mapping(msg.body) + case "trigger_start": + # msg.body is the sluggified trigger + asl_slug = msg.body + ui_id = self._trigger_reverse_map.get(asl_slug) + + if ui_id: + payload = {"type": "trigger_update", "id": ui_id, "achieved": True} + await self._send_experiment_update(payload) + self.logger.info(f"UI Update: Trigger {asl_slug} started (ID: {ui_id})") + + case "trigger_end": + asl_slug = msg.body + ui_id = self._trigger_reverse_map.get(asl_slug) + + if ui_id: + payload = {"type": "trigger_update", "id": ui_id, "achieved": False} + await self._send_experiment_update(payload) + self.logger.info(f"UI Update: Trigger {asl_slug} ended (ID: {ui_id})") case "transition_phase": new_phase_id = msg.body self.logger.info(f"Phase transition detected: {new_phase_id}") @@ -111,6 +151,10 @@ class UserInterruptAgent(BaseAgent): payload = {"type": "phase_update", "phase_id": new_phase_id} await self._send_experiment_update(payload) + case "active_norms_update": + asl_slugs = [s.strip() for s in msg.body.split(";")] + + await self._broadcast_cond_norms(asl_slugs) case _: self.logger.debug(f"Received internal message on unhandled thread: {msg.thread}") @@ -132,6 +176,54 @@ class UserInterruptAgent(BaseAgent): await asyncio.sleep(2) + async def _broadcast_cond_norms(self, active_slugs: list[str]): + """ + Sends the current state of all conditional norms to the UI. + :param active_slugs: A list of slugs (strings) currently active in the BDI core. + """ + updates = [] + + for asl_slug, ui_id in self._cond_norm_reverse_map.items(): + is_active = asl_slug in active_slugs + updates.append({"id": ui_id, "name": asl_slug, "active": is_active}) + + payload = {"type": "cond_norms_state_update", "norms": updates} + + await self._send_experiment_update(payload) + self.logger.debug(f"Broadcasted state for {len(updates)} conditional norms.") + + def _create_mapping(self, program_json: str): + try: + program = Program.model_validate_json(program_json) + self._trigger_map = {} + self._trigger_reverse_map = {} + self._goal_map = {} + self._cond_norm_map = {} + self._cond_norm_reverse_map = {} + + for phase in program.phases: + for trigger in phase.triggers: + slug = AgentSpeakGenerator.slugify(trigger) + self._trigger_map[str(trigger.id)] = slug + self._trigger_reverse_map[slug] = str(trigger.id) + + for goal in phase.goals: + self._goal_map[str(goal.id)] = AgentSpeakGenerator.slugify(goal) + + for norm in phase.conditional_norms: + if norm.condition: + asl_slug = AgentSpeakGenerator.slugify(norm) + belief_id = str(norm.condition) + + self._cond_norm_map[belief_id] = asl_slug + self._cond_norm_reverse_map[asl_slug] = belief_id + + self.logger.info( + f"Mapped {len(self._trigger_map)} triggers and {len(self._goal_map)} goals." + ) + except Exception as e: + self.logger.error(f"Mapping failed: {e}") + async def _send_experiment_update(self, data): """ Sends an update to the 'experiment' topic. @@ -194,3 +286,11 @@ class UserInterruptAgent(BaseAgent): "Sent button_override belief with id '%s' to Program manager.", belief_id, ) + + async def _send_to_bdi(self, thread: str, body: str): + """Send slug of trigger to BDI""" + msg = InternalMessage( + to=settings.agent_settings.bdi_core_name, sender=self.name, thread=thread, body=body + ) + await self.send(msg) + self.logger.info(f"Directly forced {thread} in BDI: {body}") From 45719c580bbfaf054932e36ee52997d2828372bb Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:49:13 +0100 Subject: [PATCH 36/90] feat: prepend more silence before speech audio for better transcription beginnings ref: N25B-429 --- .env.example | 2 +- src/control_backend/agents/perception/vad_agent.py | 12 +++++++----- src/control_backend/core/config.py | 2 ++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index d498054..41a382a 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,7 @@ LLM_SETTINGS__LOCAL_LLM_URL="http://localhost:1234/v1/chat/completions" LLM_SETTINGS__LOCAL_LLM_MODEL="gpt-oss" # Number of non-speech chunks to wait before speech ended. A chunk is approximately 31 ms. Increasing this number allows longer pauses in speech, but also increases response time. -BEHAVIOUR_SETTINGS__VAD_NON_SPEECH_PATIENCE_CHUNKS=3 +BEHAVIOUR_SETTINGS__VAD_NON_SPEECH_PATIENCE_CHUNKS=15 # Timeout in milliseconds for socket polling. Increase this number if network latency/jitter is high, often the case when using Wi-Fi. Perhaps 500 ms. A symptom of this issue is transcriptions getting cut off. BEHAVIOUR_SETTINGS__SOCKET_POLLER_TIMEOUT_MS=100 diff --git a/src/control_backend/agents/perception/vad_agent.py b/src/control_backend/agents/perception/vad_agent.py index 70fa9e1..e47b27a 100644 --- a/src/control_backend/agents/perception/vad_agent.py +++ b/src/control_backend/agents/perception/vad_agent.py @@ -229,10 +229,11 @@ class VADAgent(BaseAgent): assert self.model is not None prob = self.model(torch.from_numpy(chunk), settings.vad_settings.sample_rate_hz).item() non_speech_patience = settings.behaviour_settings.vad_non_speech_patience_chunks + begin_silence_length = settings.behaviour_settings.vad_begin_silence_chunks prob_threshold = settings.behaviour_settings.vad_prob_threshold if prob > prob_threshold: - if self.i_since_speech > non_speech_patience: + if self.i_since_speech > non_speech_patience + begin_silence_length: self.logger.debug("Speech started.") self.audio_buffer = np.append(self.audio_buffer, chunk) self.i_since_speech = 0 @@ -246,11 +247,12 @@ class VADAgent(BaseAgent): continue # Speech probably ended. Make sure we have a usable amount of data. - if len(self.audio_buffer) >= 3 * len(chunk): + if len(self.audio_buffer) > begin_silence_length * len(chunk): self.logger.debug("Speech ended.") assert self.audio_out_socket is not None await self.audio_out_socket.send(self.audio_buffer[: -2 * len(chunk)].tobytes()) - # At this point, we know that the speech has ended. - # Prepend the last chunk that had no speech, for a more fluent boundary - self.audio_buffer = chunk + # At this point, we know that there is no speech. + # Prepend the last few chunks that had no speech, for a more fluent boundary. + self.audio_buffer = np.append(self.audio_buffer, chunk) + self.audio_buffer = self.audio_buffer[-begin_silence_length * len(chunk) :] diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index 2ed5c04..02018ee 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -73,6 +73,7 @@ class BehaviourSettings(BaseModel): :ivar vad_prob_threshold: Probability threshold for Voice Activity Detection. :ivar vad_initial_since_speech: Initial value for 'since speech' counter in VAD. :ivar vad_non_speech_patience_chunks: Number of non-speech chunks to wait before speech ended. + :ivar vad_begin_silence_chunks: The number of chunks of silence to prepend to speech chunks. :ivar transcription_max_concurrent_tasks: Maximum number of concurrent transcription tasks. :ivar transcription_words_per_minute: Estimated words per minute for transcription timing. :ivar transcription_words_per_token: Estimated words per token for transcription timing. @@ -90,6 +91,7 @@ class BehaviourSettings(BaseModel): vad_prob_threshold: float = 0.5 vad_initial_since_speech: int = 100 vad_non_speech_patience_chunks: int = 15 + vad_begin_silence_chunks: int = 3 # transcription behaviour transcription_max_concurrent_tasks: int = 3 From 6b34f4b82cf39bf0e1a55a721f6567af177751f9 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Thu, 8 Jan 2026 10:59:24 +0100 Subject: [PATCH 37/90] fix: small bugfix ref: N25B-400 --- .../agents/user_interrupt/user_interrupt_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index b762e68..3585209 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -210,7 +210,7 @@ class UserInterruptAgent(BaseAgent): for goal in phase.goals: self._goal_map[str(goal.id)] = AgentSpeakGenerator.slugify(goal) - for norm in phase.conditional_norms: + for norm in phase.norms: if norm.condition: asl_slug = AgentSpeakGenerator.slugify(norm) belief_id = str(norm.condition) From b27e5180c46c094fb53236bff34ab312ae6b7918 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Thu, 8 Jan 2026 11:25:53 +0100 Subject: [PATCH 38/90] feat: small implementation change ref: N25B-400 --- .../agents/user_interrupt/user_interrupt_agent.py | 4 ++-- test/unit/agents/bdi/test_bdi_program_manager.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 3585209..465125d 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -106,8 +106,8 @@ class UserInterruptAgent(BaseAgent): "Forwarded button press (override) with context '%s' to BDI Core.", event_context, ) - else: - await self._send_to_program_manager(event_context) + elif asl_cond_norm := self._cond_norm_map.get(ui_id): + await self._send_to_bdi("force_norm", asl_cond_norm) self.logger.info( "Forwarded button press (override) with context '%s' to BDIProgramManager.", event_context, diff --git a/test/unit/agents/bdi/test_bdi_program_manager.py b/test/unit/agents/bdi/test_bdi_program_manager.py index 50dc4ed..644c23b 100644 --- a/test/unit/agents/bdi/test_bdi_program_manager.py +++ b/test/unit/agents/bdi/test_bdi_program_manager.py @@ -95,7 +95,9 @@ async def test_receive_programs_valid_and_invalid(): assert forwarded.phases[0].goals[0].name == "G1" # Verify history clear was triggered - assert manager._send_clear_llm_history.await_count == 1 + assert ( + manager._send_clear_llm_history.await_count == 2 + ) # first sends program to UserInterrupt, then clears LLM @pytest.mark.asyncio From 3a8d1730a1cb2bf205ba2eabaa936bcea5461dd0 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Thu, 8 Jan 2026 12:29:16 +0100 Subject: [PATCH 39/90] fix: made mapping for conditional norms only ref: N25B-400 --- src/control_backend/agents/bdi/agentspeak_generator.py | 5 +++++ .../agents/user_interrupt/user_interrupt_agent.py | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 18cb794..15a5120 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -415,6 +415,11 @@ class AgentSpeakGenerator: def slugify(element: ProgramElement) -> str: raise NotImplementedError(f"Cannot convert element {element} to a slug.") + @slugify.register + @staticmethod + def _(n: Norm) -> str: + return f"norm_{AgentSpeakGenerator._slugify_str(n.norm)}" + @slugify.register @staticmethod def _(sb: SemanticBelief) -> str: diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 465125d..925e10e 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -8,7 +8,7 @@ from control_backend.agents import BaseAgent from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings -from control_backend.schemas.program import Program +from control_backend.schemas.program import BasicNorm, Program from control_backend.schemas.ri_message import GestureCommand, RIEndpoint, SpeechCommand @@ -211,15 +211,17 @@ class UserInterruptAgent(BaseAgent): self._goal_map[str(goal.id)] = AgentSpeakGenerator.slugify(goal) for norm in phase.norms: - if norm.condition: + if not isinstance(norm, BasicNorm): asl_slug = AgentSpeakGenerator.slugify(norm) - belief_id = str(norm.condition) + + belief_id = str(norm.condition.id) self._cond_norm_map[belief_id] = asl_slug self._cond_norm_reverse_map[asl_slug] = belief_id self.logger.info( - f"Mapped {len(self._trigger_map)} triggers and {len(self._goal_map)} goals." + f"Mapped {len(self._trigger_map)} triggers and {len(self._goal_map)} goals " + f"and {len(self._cond_norm_map)} conditional norms for UserInterruptAgent." ) except Exception as e: self.logger.error(f"Mapping failed: {e}") From cc0d5af28c200eb7feec05a2bcb3c6b3a71b5dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Otgaar?= Date: Thu, 8 Jan 2026 12:56:22 +0100 Subject: [PATCH 40/90] chore: fixing bugs --- .../agents/communication/ri_communication_agent.py | 12 ++++++++++++ .../agents/user_interrupt/user_interrupt_agent.py | 7 +++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 34e5b25..45e3511 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -8,6 +8,7 @@ from zmq.asyncio import Context from control_backend.agents import BaseAgent from control_backend.agents.actuation.robot_gesture_agent import RobotGestureAgent from control_backend.core.config import settings +from control_backend.schemas.internal_message import InternalMessage from ..actuation.robot_speech_agent import RobotSpeechAgent from ..perception import VADAgent @@ -298,3 +299,14 @@ class RICommunicationAgent(BaseAgent): self.logger.debug("Restarting communication negotiation.") if await self._negotiate_connection(max_retries=1): self.connected = True + + async def handle_message(self, msg: InternalMessage): + """ + Handle an incoming message. + + Currently not implemented for this agent. + + :param msg: The received message. + :raises NotImplementedError: Always, since this method is not implemented. + """ + self.logger.warning("custom warning for handle msg in ri coms %s", self.name) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 50df979..f48e14b 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -77,6 +77,9 @@ class UserInterruptAgent(BaseAgent): event_context, ) elif event_type == "pause": + self.logger.debug( + "Received pause/resume button press with context '%s'.", event_context + ) await self._send_pause_command(event_context) if event_context: self.logger.info("Sent pause command.") @@ -175,7 +178,7 @@ class UserInterruptAgent(BaseAgent): belief_id, ) - async def _send_pause_command(self, pause: bool): + async def _send_pause_command(self, pause): """ Send a pause command to the Robot Interface via the RI Communication Agent. Send a pause command to the other internal agents; for now just VAD agent. @@ -188,7 +191,7 @@ class UserInterruptAgent(BaseAgent): ) await self.send(message) - if pause: + if pause == "true": # Send pause to VAD agent vad_message = InternalMessage( to=settings.agent_settings.vad_name, From 13605678209b23c46ff026bc56f3cefc63cba0d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Otgaar?= Date: Thu, 8 Jan 2026 13:01:38 +0100 Subject: [PATCH 41/90] chore: indenting --- src/control_backend/agents/perception/vad_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_backend/agents/perception/vad_agent.py b/src/control_backend/agents/perception/vad_agent.py index 320a849..9c5a3ef 100644 --- a/src/control_backend/agents/perception/vad_agent.py +++ b/src/control_backend/agents/perception/vad_agent.py @@ -272,7 +272,7 @@ class VADAgent(BaseAgent): # Prepend the last chunk that had no speech, for a more fluent boundary self.audio_buffer = chunk -async def handle_message(self, msg: InternalMessage): + async def handle_message(self, msg: InternalMessage): """ Handle incoming messages. From b88758fa7666b0b0e0c31f6c5f96908fc7acde28 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Thu, 8 Jan 2026 13:33:37 +0100 Subject: [PATCH 42/90] feat: phase transition independent of response ref: N25B-429 --- src/control_backend/agents/bdi/agentspeak_generator.py | 2 +- src/control_backend/agents/bdi/bdi_core_agent.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 17248a8..d02888a 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -171,7 +171,7 @@ class AgentSpeakGenerator: self._astify(to_phase) if to_phase else AstLiteral("phase", [AstString("end")]) ) - context = [from_phase_ast, ~AstLiteral("responded_this_turn")] + context = [from_phase_ast] if from_phase and from_phase.goals: context.append(self._astify(from_phase.goals[-1], achieved=True)) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 99bea80..74a747d 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -213,6 +213,14 @@ class BDICoreAgent(BaseAgent): agentspeak.runtime.Intention(), ) + # Check for transitions + self.bdi_agent.call( + agentspeak.Trigger.addition, + agentspeak.GoalType.achievement, + agentspeak.Literal("transition_phase"), + agentspeak.runtime.Intention(), + ) + self._wake_bdi_loop.set() self.logger.debug(f"Added belief {self.format_belief_string(name, args)}") From 625ef0c36543474f44a91c8641172871af255b28 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Thu, 8 Jan 2026 13:36:03 +0100 Subject: [PATCH 43/90] feat: phase transition waits for all goals ref: N25B-429 --- src/control_backend/agents/bdi/agentspeak_generator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index d02888a..22b0b8e 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -172,8 +172,9 @@ class AgentSpeakGenerator: ) context = [from_phase_ast] - if from_phase and from_phase.goals: - context.append(self._astify(from_phase.goals[-1], achieved=True)) + if from_phase: + for goal in from_phase.goals: + context.append(self._astify(goal, achieved=True)) body = [ AstStatement(StatementType.REMOVE_BELIEF, from_phase_ast), From 4d0ba6944300a14525c0aef5ab1be60cecc42d1d Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Thu, 8 Jan 2026 13:44:25 +0100 Subject: [PATCH 44/90] fix: don't re-add user_said upon phase transition ref: N25B-429 --- .../agents/bdi/agentspeak_generator.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 22b0b8e..51d3a63 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -181,17 +181,17 @@ class AgentSpeakGenerator: AstStatement(StatementType.ADD_BELIEF, to_phase_ast), ] - if from_phase: - body.extend( - [ - AstStatement( - StatementType.TEST_GOAL, AstLiteral("user_said", [AstVar("Message")]) - ), - AstStatement( - StatementType.REPLACE_BELIEF, AstLiteral("user_said", [AstVar("Message")]) - ), - ] - ) + # if from_phase: + # body.extend( + # [ + # AstStatement( + # StatementType.TEST_GOAL, AstLiteral("user_said", [AstVar("Message")]) + # ), + # AstStatement( + # StatementType.REPLACE_BELIEF, AstLiteral("user_said", [AstVar("Message")]) + # ), + # ] + # ) # Notify outside world about transition body.append( From 133019a92823646ace45a9cc9a3bf338c95af088 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Thu, 8 Jan 2026 14:04:44 +0100 Subject: [PATCH 45/90] feat: trigger name and trigger checks on belief update ref: N25B-429 --- src/control_backend/agents/bdi/bdi_core_agent.py | 8 ++++++++ src/control_backend/schemas/program.py | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 74a747d..1658ccd 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -221,6 +221,14 @@ class BDICoreAgent(BaseAgent): agentspeak.runtime.Intention(), ) + # Check triggers + self.bdi_agent.call( + agentspeak.Trigger.addition, + agentspeak.GoalType.achievement, + agentspeak.Literal("check_triggers"), + agentspeak.runtime.Intention(), + ) + self._wake_bdi_loop.set() self.logger.debug(f"Added belief {self.format_belief_string(name, args)}") diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 82c017e..3c8c7b4 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -180,7 +180,6 @@ class Trigger(ProgramElement): :ivar plan: The plan to execute. """ - name: str = "" condition: Belief plan: Plan From 500bbc2d82f7d4dea89c9202bf5014e9cf010264 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Thu, 8 Jan 2026 14:52:55 +0100 Subject: [PATCH 46/90] feat: added goal start sending functionality ref: N25B-400 --- .../agents/bdi/bdi_program_manager.py | 1 - .../user_interrupt/user_interrupt_agent.py | 31 ++++++++----------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 7899e3c..f50fcf0 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -212,7 +212,6 @@ class BDIProgramManager(BaseAgent): continue self._initialize_internal_state(program) - await self._send_program_to_user_interrupt(program) await self._send_clear_llm_history() diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 925e10e..462d94f 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -1,4 +1,3 @@ -import asyncio import json import zmq @@ -151,6 +150,15 @@ class UserInterruptAgent(BaseAgent): payload = {"type": "phase_update", "phase_id": new_phase_id} await self._send_experiment_update(payload) + case "goal_start": + goal_name = msg.body + ui_id = self._goal_reverse_map.get(goal_name) + if ui_id: + payload = {"type": "goal_update", "id": ui_id, "active": True} + await self._send_experiment_update(payload) + self.logger.info(f"UI Update: Goal {goal_name} started (ID: {ui_id})") + else: + self.logger.warning(f"Goal start received for unknown goal : {goal_name}") case "active_norms_update": asl_slugs = [s.strip() for s in msg.body.split(";")] @@ -159,23 +167,6 @@ class UserInterruptAgent(BaseAgent): case _: self.logger.debug(f"Received internal message on unhandled thread: {msg.thread}") - # moet weg!!!!! - async def test_sending_behaviour(self): - self.logger.info("Starting simple test sending behaviour...") - - while True: - try: - test_data = {"type": "heartbeat", "status": "ok"} - - await self._send_experiment_update(test_data) - - except zmq.ZMQError as ze: - self.logger.error(f"ZMQ error: {ze}") - except Exception as e: - self.logger.error(f"Error: {e}") - - await asyncio.sleep(2) - async def _broadcast_cond_norms(self, active_slugs: list[str]): """ Sends the current state of all conditional norms to the UI. @@ -209,6 +200,10 @@ class UserInterruptAgent(BaseAgent): for goal in phase.goals: self._goal_map[str(goal.id)] = AgentSpeakGenerator.slugify(goal) + self._goal_reverse_map[AgentSpeakGenerator.slugify(goal)] = str(goal.id) + + for goal, id in self._goal_reverse_map.items(): + self.logger.debug(f"Goal mapping: UI ID {goal} -> {id}") for norm in phase.norms: if not isinstance(norm, BasicNorm): From 5e2126fc21f10b2c8cfeb0af0aefa46c07663532 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Thu, 8 Jan 2026 15:05:43 +0100 Subject: [PATCH 47/90] chore: code cleanup ref: N25B-400 --- .../user_interrupt/user_interrupt_agent.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 462d94f..90f4e7a 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -36,13 +36,11 @@ class UserInterruptAgent(BaseAgent): self._trigger_map = {} self._trigger_reverse_map = {} - self._goal_map = {} - self._goal_reverse_map = {} + self._goal_map = {} # id -> sluggified goal + self._goal_reverse_map = {} # sluggified goal -> id - self._cond_norm_map = {} - self._cond_norm_reverse_map = {} - - self._belief_condition_map = {} + self._cond_norm_map = {} # id -> sluggified cond norm + self._cond_norm_reverse_map = {} # sluggified cond norm -> id async def setup(self): """ @@ -61,7 +59,6 @@ class UserInterruptAgent(BaseAgent): self.pub_socket.connect(settings.zmq_settings.internal_pub_address) self.add_behavior(self._receive_button_event()) - # self.add_behavior(self.test_sending_behaviour()) async def _receive_button_event(self): """ @@ -157,13 +154,10 @@ class UserInterruptAgent(BaseAgent): payload = {"type": "goal_update", "id": ui_id, "active": True} await self._send_experiment_update(payload) self.logger.info(f"UI Update: Goal {goal_name} started (ID: {ui_id})") - else: - self.logger.warning(f"Goal start received for unknown goal : {goal_name}") case "active_norms_update": asl_slugs = [s.strip() for s in msg.body.split(";")] await self._broadcast_cond_norms(asl_slugs) - case _: self.logger.debug(f"Received internal message on unhandled thread: {msg.thread}") @@ -184,6 +178,9 @@ class UserInterruptAgent(BaseAgent): self.logger.debug(f"Broadcasted state for {len(updates)} conditional norms.") def _create_mapping(self, program_json: str): + """ + Create mappings between UI IDs and ASL slugs for triggers, goals, and conditional norms + """ try: program = Program.model_validate_json(program_json) self._trigger_map = {} From 866d7c4958b716c773a745d433bf4f6264bf855e Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Thu, 8 Jan 2026 15:13:12 +0100 Subject: [PATCH 48/90] fix: end phase loop correctly notifies about user_said ref: N25B-429 --- src/control_backend/agents/bdi/agentspeak_generator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 51d3a63..7c70a28 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -145,7 +145,10 @@ class AgentSpeakGenerator: type=TriggerType.ADDED_BELIEF, trigger_literal=AstLiteral("user_said", [AstVar("Message")]), context=[AstLiteral("phase", [AstString("end")])], - body=[AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("reply"))], + body=[ + AstStatement(StatementType.DO_ACTION, AstLiteral("notify_user_said")), + AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("reply")), + ], ) ) From c91b9991048ae0840f54e5dbb847333295958d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Otgaar?= Date: Thu, 8 Jan 2026 15:31:44 +0100 Subject: [PATCH 49/90] chore: fix bugs and make sure connected robots work --- .../agents/actuation/robot_gesture_agent.py | 2 ++ .../communication/ri_communication_agent.py | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/control_backend/agents/actuation/robot_gesture_agent.py b/src/control_backend/agents/actuation/robot_gesture_agent.py index 4f5dd79..5e40c04 100644 --- a/src/control_backend/agents/actuation/robot_gesture_agent.py +++ b/src/control_backend/agents/actuation/robot_gesture_agent.py @@ -83,6 +83,8 @@ class RobotGestureAgent(BaseAgent): self.subsocket.close() if self.pubsocket: self.pubsocket.close() + if self.repsocket: + self.repsocket.close() await super().stop() async def handle_message(self, msg: InternalMessage): diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 45e3511..7fcd07b 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -48,6 +48,8 @@ class RICommunicationAgent(BaseAgent): self._req_socket: azmq.Socket | None = None self.pub_socket: azmq.Socket | None = None self.connected = False + self.gesture_agent: RobotGestureAgent | None = None + self.speech_agent: RobotSpeechAgent | None = None async def setup(self): """ @@ -141,6 +143,7 @@ class RICommunicationAgent(BaseAgent): # At this point, we have a valid response try: + self.logger.debug("Negotiation successful. Handling rn") await self._handle_negotiation_response(received_message) # Let UI know that we're connected topic = b"ping" @@ -189,6 +192,7 @@ class RICommunicationAgent(BaseAgent): address=addr, bind=bind, ) + self.speech_agent = robot_speech_agent robot_gesture_agent = RobotGestureAgent( settings.agent_settings.robot_gesture_name, address=addr, @@ -196,6 +200,7 @@ class RICommunicationAgent(BaseAgent): gesture_data=gesture_data, single_gesture_data=single_gesture_data, ) + self.gesture_agent = robot_gesture_agent await robot_speech_agent.start() await asyncio.sleep(0.1) # Small delay await robot_gesture_agent.start() @@ -226,6 +231,7 @@ class RICommunicationAgent(BaseAgent): while self._running: if not self.connected: await asyncio.sleep(settings.behaviour_settings.sleep_s) + self.logger.debug("Not connected, skipping ping loop iteration.") continue # We need to listen and send pings. @@ -289,15 +295,27 @@ class RICommunicationAgent(BaseAgent): # Tell UI we're disconnected. topic = b"ping" data = json.dumps(False).encode() + self.logger.debug("1") if self.pub_socket: try: + self.logger.debug("2") await asyncio.wait_for(self.pub_socket.send_multipart([topic, data]), 5) except TimeoutError: + self.logger.debug("3") self.logger.warning("Connection ping for router timed out.") # Try to reboot/renegotiate + if self.gesture_agent is not None: + await self.gesture_agent.stop() + + if self.speech_agent is not None: + await self.speech_agent.stop() + + if self.pub_socket is not None: + self.pub_socket.close() + self.logger.debug("Restarting communication negotiation.") - if await self._negotiate_connection(max_retries=1): + if await self._negotiate_connection(max_retries=2): self.connected = True async def handle_message(self, msg: InternalMessage): From 4b71981a3e7e7d70fd7cfdd1026907544d0aa700 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Mon, 12 Jan 2026 09:00:50 +0100 Subject: [PATCH 50/90] fix: some bugs and some tests ref: N25B-429 --- .../agents/bdi/bdi_core_agent.py | 1 - .../agents/bdi/bdi_program_manager.py | 10 +- .../agents/bdi/default_behavior.asl | 1 + .../agents/bdi/text_belief_extractor_agent.py | 14 +- .../communication/ri_communication_agent.py | 2 +- src/control_backend/core/config.py | 2 +- .../agents/bdi/test_bdi_program_manager.py | 4 +- .../agents/bdi/test_text_belief_extractor.py | 156 ++++++++++-------- 8 files changed, 103 insertions(+), 87 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 1658ccd..0b6fb46 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -101,7 +101,6 @@ class BDICoreAgent(BaseAgent): maybe_more_work = True while maybe_more_work: maybe_more_work = False - self.logger.debug("Stepping BDI.") if self.bdi_agent.step(): maybe_more_work = True diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 8b8c68f..13bce95 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -67,14 +67,14 @@ class BDIProgramManager(BaseAgent): await self.send(msg) - def handle_message(self, msg: InternalMessage): + async def handle_message(self, msg: InternalMessage): match msg.thread: case "transition_phase": phases = json.loads(msg.body) - self._transition_phase(phases["old"], phases["new"]) + await self._transition_phase(phases["old"], phases["new"]) - def _transition_phase(self, old: str, new: str): + async def _transition_phase(self, old: str, new: str): assert old == str(self._phase.id) if new == "end": @@ -85,8 +85,8 @@ class BDIProgramManager(BaseAgent): if str(phase.id) == new: self._phase = phase - self._send_beliefs_to_semantic_belief_extractor() - self._send_goals_to_semantic_belief_extractor() + await self._send_beliefs_to_semantic_belief_extractor() + await self._send_goals_to_semantic_belief_extractor() # Notify user interaction agent msg = InternalMessage( diff --git a/src/control_backend/agents/bdi/default_behavior.asl b/src/control_backend/agents/bdi/default_behavior.asl index 249689a..f7ae16d 100644 --- a/src/control_backend/agents/bdi/default_behavior.asl +++ b/src/control_backend/agents/bdi/default_behavior.asl @@ -1,5 +1,6 @@ norms(""). +user_said(Message) : norms(Norms) <- + .notify_user_said(Message); -user_said(Message); .reply(Message, Norms). diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index feabf40..ebd9a65 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -90,7 +90,7 @@ class TextBeliefExtractorAgent(BaseAgent): self.logger.debug("Received text from LLM: %s", msg.body) self._apply_conversation_message(ChatMessage(role="assistant", content=msg.body)) case settings.agent_settings.bdi_program_manager_name: - self._handle_program_manager_message(msg) + await self._handle_program_manager_message(msg) case _: self.logger.info("Discarding message from %s", sender) return @@ -105,7 +105,7 @@ class TextBeliefExtractorAgent(BaseAgent): length_limit = settings.behaviour_settings.conversation_history_length_limit self.conversation.messages = (self.conversation.messages + [message])[-length_limit:] - def _handle_program_manager_message(self, msg: InternalMessage): + async def _handle_program_manager_message(self, msg: InternalMessage): """ Handle a message from the program manager: extract available beliefs and goals from it. @@ -114,8 +114,10 @@ class TextBeliefExtractorAgent(BaseAgent): match msg.thread: case "beliefs": self._handle_beliefs_message(msg) + await self._infer_new_beliefs() case "goals": self._handle_goals_message(msg) + await self._infer_goal_completions() case "conversation_history": if msg.body == "reset": self._reset() @@ -141,8 +143,9 @@ class TextBeliefExtractorAgent(BaseAgent): available_beliefs = [b for b in belief_list.beliefs if isinstance(b, SemanticBelief)] self.belief_inferrer.available_beliefs = available_beliefs self.logger.debug( - "Received %d semantic beliefs from the program manager.", + "Received %d semantic beliefs from the program manager: %s", len(available_beliefs), + ", ".join(b.name for b in available_beliefs), ) def _handle_goals_message(self, msg: InternalMessage): @@ -158,8 +161,9 @@ class TextBeliefExtractorAgent(BaseAgent): available_goals = [g for g in goals_list.goals if g.can_fail] self.goal_inferrer.goals = available_goals self.logger.debug( - "Received %d failable goals from the program manager.", + "Received %d failable goals from the program manager: %s", len(available_goals), + ", ".join(g.name for g in available_goals), ) async def _user_said(self, text: str): @@ -183,6 +187,7 @@ class TextBeliefExtractorAgent(BaseAgent): new_beliefs = conversation_beliefs - self._current_beliefs if not new_beliefs: + self.logger.debug("No new beliefs detected.") return self._current_beliefs |= new_beliefs @@ -217,6 +222,7 @@ class TextBeliefExtractorAgent(BaseAgent): self._current_goal_completions[goal] = achieved if not new_achieved and not new_not_achieved: + self.logger.debug("No goal achievement changes detected.") return belief_changes = BeliefMessage( diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 443e87c..b12bac6 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -248,7 +248,7 @@ class RICommunicationAgent(BaseAgent): self._req_socket.recv_json(), timeout=seconds_to_wait_total / 2 ) - if message["endpoint"] and message["endpoint"] != "ping": + if "endpoint" in message and message["endpoint"] != "ping": self.logger.debug(f'Received message "{message}" from RI.') if "endpoint" not in message: self.logger.warning("No received endpoint in message, expected ping endpoint.") diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index 02018ee..6deb1b8 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -91,7 +91,7 @@ class BehaviourSettings(BaseModel): vad_prob_threshold: float = 0.5 vad_initial_since_speech: int = 100 vad_non_speech_patience_chunks: int = 15 - vad_begin_silence_chunks: int = 3 + vad_begin_silence_chunks: int = 6 # transcription behaviour transcription_max_concurrent_tasks: int = 3 diff --git a/test/unit/agents/bdi/test_bdi_program_manager.py b/test/unit/agents/bdi/test_bdi_program_manager.py index 50dc4ed..217e814 100644 --- a/test/unit/agents/bdi/test_bdi_program_manager.py +++ b/test/unit/agents/bdi/test_bdi_program_manager.py @@ -108,8 +108,8 @@ async def test_send_clear_llm_history(mock_settings): await manager._send_clear_llm_history() - assert manager.send.await_count == 1 - msg: InternalMessage = manager.send.await_args[0][0] + assert manager.send.await_count == 2 + msg: InternalMessage = manager.send.await_args_list[0][0][0] # Verify the content and recipient assert msg.body == "clear_history" diff --git a/test/unit/agents/bdi/test_text_belief_extractor.py b/test/unit/agents/bdi/test_text_belief_extractor.py index 176afd2..6782ba1 100644 --- a/test/unit/agents/bdi/test_text_belief_extractor.py +++ b/test/unit/agents/bdi/test_text_belief_extractor.py @@ -6,10 +6,13 @@ import httpx import pytest from control_backend.agents.bdi import TextBeliefExtractorAgent +from control_backend.agents.bdi.text_belief_extractor_agent import BeliefState from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings from control_backend.schemas.belief_list import BeliefList +from control_backend.schemas.belief_message import Belief as InternalBelief from control_backend.schemas.belief_message import BeliefMessage +from control_backend.schemas.chat_history import ChatHistory, ChatMessage from control_backend.schemas.program import ( ConditionalNorm, KeywordBelief, @@ -23,11 +26,21 @@ from control_backend.schemas.program import ( @pytest.fixture -def agent(): - agent = TextBeliefExtractorAgent("text_belief_agent") - agent.send = AsyncMock() - agent._query_llm = AsyncMock() - return agent +def llm(): + llm = TextBeliefExtractorAgent.LLM(MagicMock(), 4) + llm._query_llm = AsyncMock() + return llm + + +@pytest.fixture +def agent(llm): + with patch( + "control_backend.agents.bdi.text_belief_extractor_agent.TextBeliefExtractorAgent.LLM", + return_value=llm, + ): + agent = TextBeliefExtractorAgent("text_belief_agent") + agent.send = AsyncMock() + return agent @pytest.fixture @@ -102,24 +115,12 @@ async def test_handle_message_from_transcriber(agent, mock_settings): agent.send.assert_awaited_once() # noqa # `agent.send` has no such property, but we mock it. sent: InternalMessage = agent.send.call_args.args[0] # noqa - assert sent.to == mock_settings.agent_settings.bdi_belief_collector_name + assert sent.to == mock_settings.agent_settings.bdi_core_name assert sent.thread == "beliefs" - parsed = json.loads(sent.body) - assert parsed == {"beliefs": {"user_said": [transcription]}, "type": "belief_extraction_text"} - - -@pytest.mark.asyncio -async def test_process_user_said(agent, mock_settings): - transcription = "this is a test" - - await agent._user_said(transcription) - - agent.send.assert_awaited_once() # noqa # `agent.send` has no such property, but we mock it. - sent: InternalMessage = agent.send.call_args.args[0] # noqa - assert sent.to == mock_settings.agent_settings.bdi_belief_collector_name - assert sent.thread == "beliefs" - parsed = json.loads(sent.body) - assert parsed["beliefs"]["user_said"] == [transcription] + parsed = BeliefMessage.model_validate_json(sent.body) + replaced_last = parsed.replace.pop() + assert replaced_last.name == "user_said" + assert replaced_last.arguments == [transcription] @pytest.mark.asyncio @@ -144,46 +145,46 @@ async def test_query_llm(): "control_backend.agents.bdi.text_belief_extractor_agent.httpx.AsyncClient", return_value=mock_async_client, ): - agent = TextBeliefExtractorAgent("text_belief_agent") + llm = TextBeliefExtractorAgent.LLM(MagicMock(), 4) - res = await agent._query_llm("hello world", {"type": "null"}) + res = await llm._query_llm("hello world", {"type": "null"}) # Response content was set as "null", so should be deserialized as None assert res is None @pytest.mark.asyncio -async def test_retry_query_llm_success(agent): - agent._query_llm.return_value = None - res = await agent._retry_query_llm("hello world", {"type": "null"}) +async def test_retry_query_llm_success(llm): + llm._query_llm.return_value = None + res = await llm.query("hello world", {"type": "null"}) - agent._query_llm.assert_called_once() + llm._query_llm.assert_called_once() assert res is None @pytest.mark.asyncio -async def test_retry_query_llm_success_after_failure(agent): - agent._query_llm.side_effect = [KeyError(), "real value"] - res = await agent._retry_query_llm("hello world", {"type": "string"}) +async def test_retry_query_llm_success_after_failure(llm): + llm._query_llm.side_effect = [KeyError(), "real value"] + res = await llm.query("hello world", {"type": "string"}) - assert agent._query_llm.call_count == 2 + assert llm._query_llm.call_count == 2 assert res == "real value" @pytest.mark.asyncio -async def test_retry_query_llm_failures(agent): - agent._query_llm.side_effect = [KeyError(), KeyError(), KeyError(), "real value"] - res = await agent._retry_query_llm("hello world", {"type": "string"}) +async def test_retry_query_llm_failures(llm): + llm._query_llm.side_effect = [KeyError(), KeyError(), KeyError(), "real value"] + res = await llm.query("hello world", {"type": "string"}) - assert agent._query_llm.call_count == 3 + assert llm._query_llm.call_count == 3 assert res is None @pytest.mark.asyncio -async def test_retry_query_llm_fail_immediately(agent): - agent._query_llm.side_effect = [KeyError(), "real value"] - res = await agent._retry_query_llm("hello world", {"type": "string"}, tries=1) +async def test_retry_query_llm_fail_immediately(llm): + llm._query_llm.side_effect = [KeyError(), "real value"] + res = await llm.query("hello world", {"type": "string"}, tries=1) - assert agent._query_llm.call_count == 1 + assert llm._query_llm.call_count == 1 assert res is None @@ -192,7 +193,7 @@ async def test_extracting_semantic_beliefs(agent): """ The Program Manager sends beliefs to this agent. Test whether the agent handles them correctly. """ - assert len(agent.available_beliefs) == 0 + assert len(agent.belief_inferrer.available_beliefs) == 0 beliefs = BeliefList( beliefs=[ KeywordBelief( @@ -213,26 +214,28 @@ async def test_extracting_semantic_beliefs(agent): to=settings.agent_settings.text_belief_extractor_name, sender=settings.agent_settings.bdi_program_manager_name, body=beliefs.model_dump_json(), + thread="beliefs", ), ) - assert len(agent.available_beliefs) == 2 + assert len(agent.belief_inferrer.available_beliefs) == 2 @pytest.mark.asyncio -async def test_handle_invalid_program(agent, sample_program): - agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) - agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) - assert len(agent.available_beliefs) == 2 +async def test_handle_invalid_beliefs(agent, sample_program): + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + assert len(agent.belief_inferrer.available_beliefs) == 2 await agent.handle_message( InternalMessage( to=settings.agent_settings.text_belief_extractor_name, sender=settings.agent_settings.bdi_program_manager_name, body=json.dumps({"phases": "Invalid"}), + thread="beliefs", ), ) - assert len(agent.available_beliefs) == 2 + assert len(agent.belief_inferrer.available_beliefs) == 2 @pytest.mark.asyncio @@ -254,13 +257,13 @@ async def test_handle_robot_response(agent): @pytest.mark.asyncio -async def test_simulated_real_turn_with_beliefs(agent, sample_program): +async def test_simulated_real_turn_with_beliefs(agent, llm, sample_program): """Test sending user message to extract beliefs from.""" - agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) - agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].triggers[0].condition) # Send a user message with the belief that there's no more booze - agent._query_llm.return_value = {"is_pirate": None, "no_more_booze": True} + llm._query_llm.return_value = {"is_pirate": None, "no_more_booze": True} assert len(agent.conversation.messages) == 0 await agent.handle_message( InternalMessage( @@ -275,20 +278,20 @@ async def test_simulated_real_turn_with_beliefs(agent, sample_program): assert agent.send.call_count == 2 # First should be the beliefs message - message: InternalMessage = agent.send.call_args_list[0].args[0] + message: InternalMessage = agent.send.call_args_list[1].args[0] beliefs = BeliefMessage.model_validate_json(message.body) assert len(beliefs.create) == 1 assert beliefs.create[0].name == "no_more_booze" @pytest.mark.asyncio -async def test_simulated_real_turn_no_beliefs(agent, sample_program): +async def test_simulated_real_turn_no_beliefs(agent, llm, sample_program): """Test a user message to extract beliefs from, but no beliefs are formed.""" - agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) - agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].triggers[0].condition) # Send a user message with no new beliefs - agent._query_llm.return_value = {"is_pirate": None, "no_more_booze": None} + llm._query_llm.return_value = {"is_pirate": None, "no_more_booze": None} await agent.handle_message( InternalMessage( to=settings.agent_settings.text_belief_extractor_name, @@ -302,17 +305,17 @@ async def test_simulated_real_turn_no_beliefs(agent, sample_program): @pytest.mark.asyncio -async def test_simulated_real_turn_no_new_beliefs(agent, sample_program): +async def test_simulated_real_turn_no_new_beliefs(agent, llm, sample_program): """ Test a user message to extract beliefs from, but no new beliefs are formed because they already existed. """ - agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) - agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) - agent.beliefs["is_pirate"] = True + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + agent._current_beliefs = BeliefState(true={InternalBelief(name="is_pirate", arguments=None)}) # Send a user message with the belief the user is a pirate, still - agent._query_llm.return_value = {"is_pirate": True, "no_more_booze": None} + llm._query_llm.return_value = {"is_pirate": True, "no_more_booze": None} await agent.handle_message( InternalMessage( to=settings.agent_settings.text_belief_extractor_name, @@ -326,17 +329,19 @@ async def test_simulated_real_turn_no_new_beliefs(agent, sample_program): @pytest.mark.asyncio -async def test_simulated_real_turn_remove_belief(agent, sample_program): +async def test_simulated_real_turn_remove_belief(agent, llm, sample_program): """ Test a user message to extract beliefs from, but an existing belief is determined no longer to hold. """ - agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) - agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) - agent.beliefs["no_more_booze"] = True + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + agent._current_beliefs = BeliefState( + true={InternalBelief(name="no_more_booze", arguments=None)}, + ) # Send a user message with the belief the user is a pirate, still - agent._query_llm.return_value = {"is_pirate": None, "no_more_booze": False} + llm._query_llm.return_value = {"is_pirate": None, "no_more_booze": False} await agent.handle_message( InternalMessage( to=settings.agent_settings.text_belief_extractor_name, @@ -349,18 +354,23 @@ async def test_simulated_real_turn_remove_belief(agent, sample_program): assert agent.send.call_count == 2 # Agent's current beliefs should've changed - assert not agent.beliefs["no_more_booze"] + assert any(b.name == "no_more_booze" for b in agent._current_beliefs.false) @pytest.mark.asyncio -async def test_llm_failure_handling(agent, sample_program): +async def test_llm_failure_handling(agent, llm, sample_program): """ Check that the agent handles failures gracefully without crashing. """ - agent._query_llm.side_effect = httpx.HTTPError("") - agent.available_beliefs.append(sample_program.phases[0].norms[0].condition) - agent.available_beliefs.append(sample_program.phases[0].triggers[0].condition) + llm._query_llm.side_effect = httpx.HTTPError("") + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].norms[0].condition) + agent.belief_inferrer.available_beliefs.append(sample_program.phases[0].triggers[0].condition) - belief_changes = await agent._infer_turn() + belief_changes = await agent.belief_inferrer.infer_from_conversation( + ChatHistory( + messages=[ChatMessage(role="user", content="Good day!")], + ), + ) - assert len(belief_changes) == 0 + assert len(belief_changes.true) == 0 + assert len(belief_changes.false) == 0 From d202abcd1bd87d31b18ec24f7d920836645ed4ea Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Mon, 12 Jan 2026 12:51:24 +0100 Subject: [PATCH 51/90] fix: phases update correctly there was a bug where phases would not update without restarting cb ref: N25B-400 --- .../agents/bdi/bdi_program_manager.py | 10 ++++----- .../user_interrupt/user_interrupt_agent.py | 21 ------------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index f50fcf0..092a2c6 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -67,14 +67,14 @@ class BDIProgramManager(BaseAgent): await self.send(msg) - def handle_message(self, msg: InternalMessage): + async def handle_message(self, msg: InternalMessage): match msg.thread: case "transition_phase": phases = json.loads(msg.body) - self._transition_phase(phases["old"], phases["new"]) + await self._transition_phase(phases["old"], phases["new"]) - def _transition_phase(self, old: str, new: str): + async def _transition_phase(self, old: str, new: str): if old != str(self._phase.id): self.logger.warning( f"Phase transition desync detected! ASL requested move from '{old}', " @@ -90,8 +90,8 @@ class BDIProgramManager(BaseAgent): if str(phase.id) == new: self._phase = phase - self._send_beliefs_to_semantic_belief_extractor() - self._send_goals_to_semantic_belief_extractor() + await self._send_beliefs_to_semantic_belief_extractor() + await self._send_goals_to_semantic_belief_extractor() # Notify user interaction agent msg = InternalMessage( diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 90f4e7a..c42449a 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -260,27 +260,6 @@ class UserInterruptAgent(BaseAgent): ) await self.send(out_msg) - async def _send_to_program_manager(self, belief_id: str): - """ - Send a button_override belief to the BDIProgramManager. - - :param belief_id: The belief_id that overrides the goal/trigger/conditional norm. - this id can belong to a basic belief or an inferred belief. - See also: https://utrechtuniversity.youtrack.cloud/articles/N25B-A-27/UI-components - """ - data = {"belief": belief_id} - message = InternalMessage( - to=settings.agent_settings.bdi_program_manager_name, - sender=self.name, - body=json.dumps(data), - thread="belief_override_id", - ) - await self.send(message) - self.logger.info( - "Sent button_override belief with id '%s' to Program manager.", - belief_id, - ) - async def _send_to_bdi(self, thread: str, body: str): """Send slug of trigger to BDI""" msg = InternalMessage( From 54c835cc0fc4d53320dc2ffa8d09b101b4f92f65 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Mon, 12 Jan 2026 15:37:04 +0100 Subject: [PATCH 52/90] feat: added force_norm handling in BDI core agent ref: N25B-400 --- src/control_backend/agents/bdi/bdi_core_agent.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 7f961ec..572786c 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -164,6 +164,8 @@ class BDICoreAgent(BaseAgent): self._set_goal("transition_phase") case "force_trigger": self._force_trigger(msg.body) + case "force_norm": + self._force_norm(msg.body) case _: self.logger.warning("Received unknow user interruption: %s", msg) @@ -311,6 +313,17 @@ class BDICoreAgent(BaseAgent): self.logger.info("Manually forced trigger %s.", name) + # TODO: make this compatible for critical norms + def _force_norm(self, name: str): + self.bdi_agent.call( + agentspeak.Trigger.addition, + agentspeak.GoalType.belief, + agentspeak.Literal("force_norm", (agentspeak.Literal(agentspeak.asl_repr(name)),)), + agentspeak.runtime.Intention(), + ) + + self.logger.info("Manually forced norm %s.", name) + def _add_custom_actions(self) -> None: """ Add any custom actions here. Inside `@self.actions.add()`, the first argument is From 4e113c2d5c2b6f2440f5b96f2489870de0a9f6e6 Mon Sep 17 00:00:00 2001 From: Kasper Date: Mon, 12 Jan 2026 16:20:24 +0100 Subject: [PATCH 53/90] fix: default plan and norm force ref: N25B-400 --- .../agents/bdi/bdi_core_agent.py | 2 +- .../agents/bdi/default_behavior.asl | 17 +++++++++++----- .../user_interrupt/user_interrupt_agent.py | 20 +++++++++++-------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 572786c..44e9a63 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -318,7 +318,7 @@ class BDICoreAgent(BaseAgent): self.bdi_agent.call( agentspeak.Trigger.addition, agentspeak.GoalType.belief, - agentspeak.Literal("force_norm", (agentspeak.Literal(agentspeak.asl_repr(name)),)), + agentspeak.Literal(f"force_{name}"), agentspeak.runtime.Intention(), ) diff --git a/src/control_backend/agents/bdi/default_behavior.asl b/src/control_backend/agents/bdi/default_behavior.asl index f7ae16d..f7d1f95 100644 --- a/src/control_backend/agents/bdi/default_behavior.asl +++ b/src/control_backend/agents/bdi/default_behavior.asl @@ -1,6 +1,13 @@ -norms(""). +norm("Be friendly"). -+user_said(Message) : norms(Norms) <- - .notify_user_said(Message); - -user_said(Message); - .reply(Message, Norms). ++!reply + : user_said(Message) + <- .findall(Norm, norm(Norm), Norms); + .reply(Message, Norms). + ++user_said(Message) + <- .notify_user_said(Message); + !reply. + ++!transition_phase <- true. ++!check_triggers <- true. \ No newline at end of file diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index c42449a..e6ba463 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -7,7 +7,7 @@ from control_backend.agents import BaseAgent from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings -from control_backend.schemas.program import BasicNorm, Program +from control_backend.schemas.program import ConditionalNorm, Program from control_backend.schemas.ri_message import GestureCommand, RIEndpoint, SpeechCommand @@ -82,6 +82,8 @@ class UserInterruptAgent(BaseAgent): self.logger.error("Received invalid JSON payload on topic %s", topic) continue + self.logger.debug("Received event type %s", event_type) + if event_type == "speech": await self._send_to_speech_agent(event_context) self.logger.info( @@ -108,6 +110,8 @@ class UserInterruptAgent(BaseAgent): "Forwarded button press (override) with context '%s' to BDIProgramManager.", event_context, ) + else: + self.logger.warning("Could not determine which element to override.") else: self.logger.warning( "Received button press with unknown type '%s' (context: '%s').", @@ -203,13 +207,15 @@ class UserInterruptAgent(BaseAgent): self.logger.debug(f"Goal mapping: UI ID {goal} -> {id}") for norm in phase.norms: - if not isinstance(norm, BasicNorm): + if isinstance(norm, ConditionalNorm): asl_slug = AgentSpeakGenerator.slugify(norm) - belief_id = str(norm.condition.id) + norm_id = str(norm.id) - self._cond_norm_map[belief_id] = asl_slug - self._cond_norm_reverse_map[asl_slug] = belief_id + self._cond_norm_map[norm_id] = asl_slug + self._cond_norm_reverse_map[asl_slug] = norm_id + self._cond_norm_reverse_map[asl_slug] = norm_id + self.logger.debug("Added conditional norm %s", asl_slug) self.logger.info( f"Mapped {len(self._trigger_map)} triggers and {len(self._goal_map)} goals " @@ -262,8 +268,6 @@ class UserInterruptAgent(BaseAgent): async def _send_to_bdi(self, thread: str, body: str): """Send slug of trigger to BDI""" - msg = InternalMessage( - to=settings.agent_settings.bdi_core_name, sender=self.name, thread=thread, body=body - ) + msg = InternalMessage(to=settings.agent_settings.bdi_core_name, thread=thread, body=body) await self.send(msg) self.logger.info(f"Directly forced {thread} in BDI: {body}") From 0f0927647701a81c4faa7a6d2c069649b9ddbd0e Mon Sep 17 00:00:00 2001 From: Kasper Date: Mon, 12 Jan 2026 17:02:39 +0100 Subject: [PATCH 54/90] fix: send norms back to UI ref: N25B-400 --- src/control_backend/agents/bdi/agentspeak_generator.py | 7 ++++++- src/control_backend/agents/bdi/bdi_core_agent.py | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index f90e63e..9ab409d 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -3,6 +3,7 @@ from functools import singledispatchmethod from slugify import slugify from control_backend.agents.bdi.agentspeak_ast import ( + AstAtom, AstBinaryOp, AstExpression, AstLiteral, @@ -215,7 +216,11 @@ class AgentSpeakGenerator: match norm: case ConditionalNorm(condition=cond): - rule = AstRule(self._astify(norm), self._astify(phase) & self._astify(cond)) + rule = AstRule( + self._astify(norm), + self._astify(phase) & self._astify(cond) + | AstAtom(f"force_{self.slugify(norm)}"), + ) case BasicNorm(): rule = AstRule(self._astify(norm), self._astify(phase)) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 44e9a63..206e411 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -341,7 +341,6 @@ class BDICoreAgent(BaseAgent): norm_update_message = InternalMessage( to=settings.agent_settings.user_interrupt_name, - sender=self.name, thread="active_norms_update", body=str(norms), ) @@ -364,6 +363,14 @@ class BDICoreAgent(BaseAgent): norms = agentspeak.grounded(term.args[1], intention.scope) goal = agentspeak.grounded(term.args[2], intention.scope) + norm_update_message = InternalMessage( + to=settings.agent_settings.user_interrupt_name, + thread="active_norms_update", + body=str(norms), + ) + + self.add_behavior(self.send(norm_update_message)) + self.logger.debug( '"reply_with_goal" action called with message=%s, norms=%s, goal=%s', message_text, From c45a258b2283341a9c0b0e666e0937961bfe7f42 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Mon, 12 Jan 2026 19:07:05 +0100 Subject: [PATCH 55/90] fix: fixed a bug where norms where not updated Now in UserInterruptAgent we store the norm.norm and not the slugified norm ref: N25B-400 --- src/control_backend/agents/bdi/bdi_core_agent.py | 3 +-- .../agents/user_interrupt/user_interrupt_agent.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 206e411..5b24c5d 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -342,7 +342,7 @@ class BDICoreAgent(BaseAgent): norm_update_message = InternalMessage( to=settings.agent_settings.user_interrupt_name, thread="active_norms_update", - body=str(norms), + body=norms, ) self.add_behavior(self.send(norm_update_message)) @@ -362,7 +362,6 @@ class BDICoreAgent(BaseAgent): message_text = agentspeak.grounded(term.args[0], intention.scope) norms = agentspeak.grounded(term.args[1], intention.scope) goal = agentspeak.grounded(term.args[2], intention.scope) - norm_update_message = InternalMessage( to=settings.agent_settings.user_interrupt_name, thread="active_norms_update", diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index e6ba463..28ddeca 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -159,9 +159,10 @@ class UserInterruptAgent(BaseAgent): await self._send_experiment_update(payload) self.logger.info(f"UI Update: Goal {goal_name} started (ID: {ui_id})") case "active_norms_update": - asl_slugs = [s.strip() for s in msg.body.split(";")] + self.logger.info(f"msg.bodyy{msg.body}") + norm_list = [s.strip("() '\",") for s in msg.body.split(",") if s.strip("() '\",")] - await self._broadcast_cond_norms(asl_slugs) + await self._broadcast_cond_norms(norm_list) case _: self.logger.debug(f"Received internal message on unhandled thread: {msg.thread}") @@ -213,8 +214,7 @@ class UserInterruptAgent(BaseAgent): norm_id = str(norm.id) self._cond_norm_map[norm_id] = asl_slug - self._cond_norm_reverse_map[asl_slug] = norm_id - self._cond_norm_reverse_map[asl_slug] = norm_id + self._cond_norm_reverse_map[norm.norm] = norm_id self.logger.debug("Added conditional norm %s", asl_slug) self.logger.info( From 72c2c57f260c9bf05d2c88822774e206d967013f Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Mon, 12 Jan 2026 19:31:50 +0100 Subject: [PATCH 56/90] chore: merged button functionality and fix bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit merged björns branch that has the following button functionality -Pause/resume -Next phase -Restart phase -reset experiment fix bug where norms where not properly sent to the user interrupt agent ref: N25B-400 --- .../user_interrupt/user_interrupt_agent.py | 84 ++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 28ddeca..4dee823 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -8,7 +8,12 @@ from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings from control_backend.schemas.program import ConditionalNorm, Program -from control_backend.schemas.ri_message import GestureCommand, RIEndpoint, SpeechCommand +from control_backend.schemas.ri_message import ( + GestureCommand, + PauseCommand, + RIEndpoint, + SpeechCommand, +) class UserInterruptAgent(BaseAgent): @@ -70,6 +75,9 @@ class UserInterruptAgent(BaseAgent): - type: "speech", context: string that the robot has to say. - type: "gesture", context: single gesture name that the robot has to perform. - type: "override", context: belief_id that overrides the goal/trigger/conditional norm. + - type: "pause", context: boolean indicating whether to pause + - type: "reset_phase", context: None, indicates to the BDI Core to + - type: "reset_experiment", context: None, indicates to the BDI Core to """ while True: topic, body = await self.sub_socket.recv_multipart() @@ -112,6 +120,18 @@ class UserInterruptAgent(BaseAgent): ) else: self.logger.warning("Could not determine which element to override.") + elif event_type == "pause": + self.logger.debug( + "Received pause/resume button press with context '%s'.", event_context + ) + await self._send_pause_command(event_context) + if event_context: + self.logger.info("Sent pause command.") + else: + self.logger.info("Sent resume command.") + + elif event_type in ["next_phase", "reset_phase", "reset_experiment"]: + await self._send_experiment_control_to_bdi_core(event_type) else: self.logger.warning( "Received button press with unknown type '%s' (context: '%s').", @@ -271,3 +291,65 @@ class UserInterruptAgent(BaseAgent): msg = InternalMessage(to=settings.agent_settings.bdi_core_name, thread=thread, body=body) await self.send(msg) self.logger.info(f"Directly forced {thread} in BDI: {body}") + + async def _send_experiment_control_to_bdi_core(self, type): + """ + method to send experiment control buttons to bdi core. + + :param type: the type of control button we should send to the bdi core. + """ + # Switch which thread we should send to bdi core + thread = "" + match type: + case "next_phase": + thread = "force_next_phase" + case "reset_phase": + thread = "reset_current_phase" + case "reset_experiment": + thread = "reset_experiment" + case _: + self.logger.warning( + "Received unknown experiment control type '%s' to send to BDI Core.", + type, + ) + + out_msg = InternalMessage( + to=settings.agent_settings.bdi_core_name, + sender=self.name, + thread=thread, + body="", + ) + self.logger.debug("Sending experiment control '%s' to BDI Core.", thread) + await self.send(out_msg) + + async def _send_pause_command(self, pause): + """ + Send a pause command to the Robot Interface via the RI Communication Agent. + Send a pause command to the other internal agents; for now just VAD agent. + """ + cmd = PauseCommand(data=pause) + message = InternalMessage( + to=settings.agent_settings.ri_communication_name, + sender=self.name, + body=cmd.model_dump_json(), + ) + await self.send(message) + + if pause == "true": + # Send pause to VAD agent + vad_message = InternalMessage( + to=settings.agent_settings.vad_name, + sender=self.name, + body="PAUSE", + ) + await self.send(vad_message) + self.logger.info("Sent pause command to VAD Agent and RI Communication Agent.") + else: + # Send resume to VAD agent + vad_message = InternalMessage( + to=settings.agent_settings.vad_name, + sender=self.name, + body="RESUME", + ) + await self.send(vad_message) + self.logger.info("Sent resume command to VAD Agent and RI Communication Agent.") From d499111ea45ee6590149eebf8293379569409945 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Tue, 13 Jan 2026 00:52:04 +0100 Subject: [PATCH 57/90] feat: added pause functionality Storms code wasnt fully included in Bjorns branch ref: N25B-400 --- .../communication/ri_communication_agent.py | 17 ++++++++--------- .../user_interrupt/user_interrupt_agent.py | 1 - 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 2377421..719053c 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -3,12 +3,14 @@ import json import zmq import zmq.asyncio as azmq +from pydantic import ValidationError from zmq.asyncio import Context from control_backend.agents import BaseAgent from control_backend.agents.actuation.robot_gesture_agent import RobotGestureAgent from control_backend.core.config import settings from control_backend.schemas.internal_message import InternalMessage +from control_backend.schemas.ri_message import PauseCommand from ..actuation.robot_speech_agent import RobotSpeechAgent from ..perception import VADAgent @@ -320,12 +322,9 @@ class RICommunicationAgent(BaseAgent): self.connected = True async def handle_message(self, msg: InternalMessage): - """ - Handle an incoming message. - - Currently not implemented for this agent. - - :param msg: The received message. - :raises NotImplementedError: Always, since this method is not implemented. - """ - self.logger.warning("custom warning for handle msg in ri coms %s", self.name) + try: + pause_command = PauseCommand.model_validate_json(msg.body) + self._req_socket.send_json(pause_command.model_dump()) + self.logger.debug(self._req_socket.recv_json()) + except ValidationError: + self.logger.warning("Incorrect message format for PauseCommand.") diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 4dee823..05af28a 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -179,7 +179,6 @@ class UserInterruptAgent(BaseAgent): await self._send_experiment_update(payload) self.logger.info(f"UI Update: Goal {goal_name} started (ID: {ui_id})") case "active_norms_update": - self.logger.info(f"msg.bodyy{msg.body}") norm_list = [s.strip("() '\",") for s in msg.body.split(",") if s.strip("() '\",")] await self._broadcast_cond_norms(norm_list) From 0df60404449e8ee925e9c95b04c4f03c402df5d3 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Tue, 13 Jan 2026 11:24:35 +0100 Subject: [PATCH 58/90] feat: added sending goal overwrites in Userinter. ref: N25B-400 --- .../agents/user_interrupt/user_interrupt_agent.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 05af28a..708e3e5 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -118,6 +118,12 @@ class UserInterruptAgent(BaseAgent): "Forwarded button press (override) with context '%s' to BDIProgramManager.", event_context, ) + elif asl_goal := self._goal_map.get(ui_id): + await self._send_to_bdi("complete_goal", asl_goal) + self.logger.info( + "Forwarded button press (override) with context '%s' to BDI Core.", + event_context, + ) else: self.logger.warning("Could not determine which element to override.") elif event_type == "pause": From 177e844349df6b2ebebe2f4408616befcd926f2b Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Tue, 13 Jan 2026 11:46:17 +0100 Subject: [PATCH 59/90] feat: send achieved goal from interrupt->manager->semantic ref: N25B-400 --- .../agents/bdi/bdi_program_manager.py | 54 ++++++++++++++++--- .../user_interrupt/user_interrupt_agent.py | 8 +++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 092a2c6..7e6dfc0 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -42,6 +42,16 @@ class BDIProgramManager(BaseAgent): def _initialize_internal_state(self, program: Program): self._program = program self._phase = program.phases[0] # start in first phase + self._goal_mapping: dict[str, Goal] = {} + for phase in program.phases: + for goal in phase.goals: + self._populate_goal_mapping_with_goal(goal) + + def _populate_goal_mapping_with_goal(self, goal: Goal): + self._goal_mapping[str(goal.id)] = goal + for step in goal.plan.steps: + if isinstance(step, Goal): + self._populate_goal_mapping_with_goal(step) async def _create_agentspeak_and_send_to_bdi(self, program: Program): """ @@ -73,6 +83,9 @@ class BDIProgramManager(BaseAgent): phases = json.loads(msg.body) await self._transition_phase(phases["old"], phases["new"]) + case "achieve_goal": + goal_id = msg.body + self._send_achieved_goal_to_semantic_belief_extractor(goal_id) async def _transition_phase(self, old: str, new: str): if old != str(self._phase.id): @@ -138,6 +151,19 @@ class BDIProgramManager(BaseAgent): await self.send(message) + @staticmethod + def _extract_goals_from_goal(goal: Goal) -> list[Goal]: + """ + Extract all goals from a given goal, that is: the goal itself and any subgoals. + + :return: All goals within and including the given goal. + """ + goals: list[Goal] = [goal] + for plan in goal.plan: + if isinstance(plan, Goal): + goals.extend(BDIProgramManager._extract_goals_from_goal(plan)) + return goals + def _extract_current_goals(self) -> list[Goal]: """ Extract all goals from the program, including subgoals. @@ -146,15 +172,8 @@ class BDIProgramManager(BaseAgent): """ goals: list[Goal] = [] - def extract_goals_from_goal(goal_: Goal) -> list[Goal]: - goals_: list[Goal] = [goal] - for plan in goal_.plan: - if isinstance(plan, Goal): - goals_.extend(extract_goals_from_goal(plan)) - return goals_ - for goal in self._phase.goals: - goals.extend(extract_goals_from_goal(goal)) + goals.extend(self._extract_goals_from_goal(goal)) return goals @@ -173,6 +192,25 @@ class BDIProgramManager(BaseAgent): await self.send(message) + async def _send_achieved_goal_to_semantic_belief_extractor(self, achieved_goal_id: str): + """ + Inform the semantic belief extractor when a goal is marked achieved. + + :param achieved_goal_id: The id of the achieved goal. + """ + goal = self._goal_mapping.get(achieved_goal_id) + if goal is None: + self.logger.debug(f"Goal with ID {achieved_goal_id} marked achieved but was not found.") + return + + goals = self._extract_goals_from_goal(goal) + message = InternalMessage( + to=settings.agent_settings.text_belief_extractor_name, + body=GoalList(goals=goals).model_dump_json(), + thread="achieved_goals", + ) + await self.send(message) + async def _send_clear_llm_history(self): """ Clear the LLM Agent's conversation history. diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 708e3e5..d92a071 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -124,6 +124,14 @@ class UserInterruptAgent(BaseAgent): "Forwarded button press (override) with context '%s' to BDI Core.", event_context, ) + + goal_achieve_msg = InternalMessage( + to=settings.agent_settings.bdi_program_manager_name, + thread="achieve_goal", + body=ui_id, + ) + + await self.send(goal_achieve_msg) else: self.logger.warning("Could not determine which element to override.") elif event_type == "pause": From 65e0b2d250cd4a68fe0b5e2b8e7a9fadf3f150c4 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Tue, 13 Jan 2026 12:05:20 +0100 Subject: [PATCH 60/90] feat: added correct message ref: N25B-400 --- .../user_interrupt/user_interrupt_agent.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index d92a071..58d2024 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -7,6 +7,7 @@ from control_backend.agents import BaseAgent from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings +from control_backend.schemas.belief_message import Belief, BeliefMessage from control_backend.schemas.program import ConditionalNorm, Program from control_backend.schemas.ri_message import ( GestureCommand, @@ -119,7 +120,7 @@ class UserInterruptAgent(BaseAgent): event_context, ) elif asl_goal := self._goal_map.get(ui_id): - await self._send_to_bdi("complete_goal", asl_goal) + await self._send_to_bdi_belief(asl_goal) self.logger.info( "Forwarded button press (override) with context '%s' to BDI Core.", event_context, @@ -134,6 +135,9 @@ class UserInterruptAgent(BaseAgent): await self.send(goal_achieve_msg) else: self.logger.warning("Could not determine which element to override.") + self.logger.warning(self._goal_map) + self.loger.warning(ui_id) + elif event_type == "pause": self.logger.debug( "Received pause/resume button press with context '%s'.", event_context @@ -305,6 +309,20 @@ class UserInterruptAgent(BaseAgent): await self.send(msg) self.logger.info(f"Directly forced {thread} in BDI: {body}") + async def _send_to_bdi_belief(self, asl_goal: str): + """Send belief to BDI Core""" + belief_name = f"achieved_{asl_goal}" + belief = Belief(name=belief_name) + self.logger.debug(f"Sending belief to BDI Core: {belief_name}") + belief_message = BeliefMessage(create=[belief]) + msg = InternalMessage( + to=settings.agent_settings.bdi_core_name, + thread="belief_update", + body=belief_message.model_dump_json(), + ) + await self.send(msg) + self.logger.info(f"Sent belief to BDI Core: {msg}") + async def _send_experiment_control_to_bdi_core(self, type): """ method to send experiment control buttons to bdi core. From f87651f691f0b813122decf6725d264623fb13d4 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Tue, 13 Jan 2026 12:26:18 +0100 Subject: [PATCH 61/90] fix: achieved goal in bdi core ref: N25B-400 --- src/control_backend/agents/bdi/bdi_program_manager.py | 2 +- .../agents/user_interrupt/user_interrupt_agent.py | 7 ++----- src/control_backend/api/v1/endpoints/robot.py | 1 - src/control_backend/schemas/belief_message.py | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 7e6dfc0..25b7364 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -85,7 +85,7 @@ class BDIProgramManager(BaseAgent): await self._transition_phase(phases["old"], phases["new"]) case "achieve_goal": goal_id = msg.body - self._send_achieved_goal_to_semantic_belief_extractor(goal_id) + await self._send_achieved_goal_to_semantic_belief_extractor(goal_id) async def _transition_phase(self, old: str, new: str): if old != str(self._phase.id): diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 58d2024..4bf681a 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -32,7 +32,7 @@ class UserInterruptAgent(BaseAgent): Prioritized actions clear the current RI queue before inserting the new item, ensuring they are executed immediately after Pepper's current action has been fulfilled. - :ivar sub_socket: The ZMQ SUB socket used to receive user intterupts. + :ivar sub_socket: The ZMQ SUB socket used to receive user interrupts. """ def __init__(self, **kwargs): @@ -135,8 +135,6 @@ class UserInterruptAgent(BaseAgent): await self.send(goal_achieve_msg) else: self.logger.warning("Could not determine which element to override.") - self.logger.warning(self._goal_map) - self.loger.warning(ui_id) elif event_type == "pause": self.logger.debug( @@ -317,11 +315,10 @@ class UserInterruptAgent(BaseAgent): belief_message = BeliefMessage(create=[belief]) msg = InternalMessage( to=settings.agent_settings.bdi_core_name, - thread="belief_update", + thread="beliefs", body=belief_message.model_dump_json(), ) await self.send(msg) - self.logger.info(f"Sent belief to BDI Core: {msg}") async def _send_experiment_control_to_bdi_core(self, type): """ diff --git a/src/control_backend/api/v1/endpoints/robot.py b/src/control_backend/api/v1/endpoints/robot.py index afbf1ac..95a9c40 100644 --- a/src/control_backend/api/v1/endpoints/robot.py +++ b/src/control_backend/api/v1/endpoints/robot.py @@ -137,7 +137,6 @@ async def ping_stream(request: Request): logger.info("Client disconnected from SSE") break - logger.debug(f"Yielded new connection event in robot ping router: {str(connected)}") connectedJson = json.dumps(connected) yield (f"data: {connectedJson}\n\n") diff --git a/src/control_backend/schemas/belief_message.py b/src/control_backend/schemas/belief_message.py index 51411b3..226833e 100644 --- a/src/control_backend/schemas/belief_message.py +++ b/src/control_backend/schemas/belief_message.py @@ -11,7 +11,7 @@ class Belief(BaseModel): """ name: str - arguments: list[str] | None + arguments: list[str] | None = None # To make it hashable model_config = {"frozen": True} From 1c88ae60784dd41c78da6b8054fbe9cf9ab7e8c7 Mon Sep 17 00:00:00 2001 From: Storm Date: Tue, 13 Jan 2026 12:41:18 +0100 Subject: [PATCH 62/90] feat: visual emotion recognition agent ref: N25B-393 --- pyproject.toml | 1 + .../visual_emotion_recognition_agent.py | 50 + .../visual_emotion_recognizer.py | 35 + src/control_backend/core/config.py | 1 + uv.lock | 866 +++++++++++++++++- 5 files changed, 923 insertions(+), 30 deletions(-) create mode 100644 src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py create mode 100644 src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py diff --git a/pyproject.toml b/pyproject.toml index e57a03c..c06f0eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.13" dependencies = [ "agentspeak>=0.2.2", "colorlog>=6.10.1", + "deepface>=0.0.96", "fastapi[all]>=0.115.6", "mlx-whisper>=0.4.3 ; sys_platform == 'darwin'", "numpy>=2.3.3", diff --git a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py new file mode 100644 index 0000000..f301c6a --- /dev/null +++ b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py @@ -0,0 +1,50 @@ +import asyncio +import zmq +import zmq.asyncio as azmq + +from control_backend.agents import BaseAgent +from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognizer import DeepFaceEmotionRecognizer +from control_backend.core.agent_system import InternalMessage +from control_backend.core.config import settings + +# START FROM RI? + +class VisualEmotionRecognitionAgent(BaseAgent): + def __init__(self, socket_address: str, socket_bind: bool = False, timeout_ms: int = 1000): + super().__init__(settings.agent_settings.visual_emotion_recognition_name) + self.socket_address = socket_address + self.socket_bind = socket_bind + self.timeout_ms = timeout_ms + + async def setup(self): + self.logger.info("Setting up %s.", self.name) + + self.emotion_recognizer = DeepFaceEmotionRecognizer() + + self.video_in_socket = azmq.Context.instance().socket(zmq.SUB) + + if self.socket_bind: + self.video_in_socket.bind(self.socket_address) + else: + self.video_in_socket.connect(self.socket_address) + + self.video_in_socket.setsockopt_string(zmq.SUBSCRIBE, "") + self.video_in_socket.setsockopt(zmq.RCVTIMEO, self.timeout_ms) + self.video_in_socket.setsockopt(zmq.CONFLATE, 1) + + self.add_behavior(self.retrieve_frame()) + + async def retrieve_frame(self): + """ + Retrieve a video frame from the input socket. + + :return: The received video frame, or None if timeout occurs. + """ + await asyncio.sleep(1) # Yield control to the event loop + try: + frame = await self.video_in_socket.recv() + # detected_emotions contains a list of dictionaries as follows: + detected_emotions = self.emotion_recognizer.detect(frame) + except zmq.Again: + self.logger.debug("No video frame received within timeout.") + return None \ No newline at end of file diff --git a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py new file mode 100644 index 0000000..069441e --- /dev/null +++ b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py @@ -0,0 +1,35 @@ +import abc +from deepface import DeepFace +import numpy as np + +class VisualEmotionRecognizer(abc.ABC): + @abc.abstractmethod + def load_model(self): + """Load the visual emotion recognition model into memory.""" + pass + + @abc.abstractmethod + def detect(self, image): + """Recognize emotion from the given image. + + :param image: The input image for emotion recognition. + :return: Detected emotion label. + """ + pass + +class DeepFaceEmotionRecognizer(VisualEmotionRecognizer): + def __init__(self): + self.load_model() + + def load_model(self): + # Initialize DeepFace model for emotion recognition + print("Loading Deepface Emotion Model...") + dummy_img = np.zeros((224, 224, 3), dtype=np.uint8) + # analyze does not take a model as an argument, calling it once on a dummy image to load + # the model + DeepFace.analyze(dummy_img, actions=['emotion'], enforce_detection=False) + print("Deepface Emotion Model loaded.") + + def detect(self, image): + analysis = DeepFace.analyze(image, actions=['emotion'], enforce_detection=False) + return analysis['dominant_emotion'] \ No newline at end of file diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index c4a4db7..f5c1979 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -52,6 +52,7 @@ class AgentSettings(BaseModel): bdi_core_name: str = "bdi_core_agent" bdi_belief_collector_name: str = "belief_collector_agent" bdi_program_manager_name: str = "bdi_program_manager_agent" + visual_emotion_recognition_name: str = "visual_emotion_recognition_agent" text_belief_extractor_name: str = "text_belief_extractor_agent" vad_name: str = "vad_agent" llm_name: str = "llm_agent" diff --git a/uv.lock b/uv.lock index ff4b8a7..f771cd4 100644 --- a/uv.lock +++ b/uv.lock @@ -2,8 +2,21 @@ version = 1 revision = 3 requires-python = ">=3.13" resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version < '3.14'", + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.14' and sys_platform == 'darwin'", + "python_full_version < '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')", +] + +[[package]] +name = "absl-py" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588, upload-time = "2025-07-03T09:31:44.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" }, ] [[package]] @@ -49,6 +62,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, ] +[[package]] +name = "astunparse" +version = "1.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, + { name = "wheel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290, upload-time = "2019-12-22T18:12:13.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8", size = 12732, upload-time = "2019-12-22T18:12:11.297Z" }, +] + [[package]] name = "babel" version = "2.17.0" @@ -58,6 +84,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, ] +[[package]] +name = "beautifulsoup4" +version = "4.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, +] + +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + [[package]] name = "certifi" version = "2025.10.5" @@ -258,6 +306,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/04/642c1d8a448ae5ea1369eac8495740a79eb4e581a9fb0cbdce56bbf56da1/coverage-7.11.0-py3-none-any.whl", hash = "sha256:4b7589765348d78fb4e5fb6ea35d07564e387da2fc5efff62e0222971f155f68", size = 207761, upload-time = "2025-10-15T15:15:06.439Z" }, ] +[[package]] +name = "deepface" +version = "0.0.96" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fire" }, + { name = "flask" }, + { name = "flask-cors" }, + { name = "gdown" }, + { name = "gunicorn" }, + { name = "keras" }, + { name = "lightphe" }, + { name = "mtcnn" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "pandas" }, + { name = "pillow" }, + { name = "requests" }, + { name = "retina-face" }, + { name = "tensorflow" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/20/d2a2dd919bbb9aeacf20788613f4939016dc0db3e2ebfbeff647c5e2299e/deepface-0.0.96.tar.gz", hash = "sha256:d3e7a418a48c8230621a06200f017599fbddfee0f1d2c5c8382299c7b67e6ef6", size = 112346, upload-time = "2025-11-23T14:16:29.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/19/282ae42de09fc0e29ecfc902453315b06a22eb273782d5527223232f9949/deepface-0.0.96-py3-none-any.whl", hash = "sha256:957a30f8dbbce2ace4cea89bd16038b2a776a8c6146cf61138beba79537adf69", size = 133122, upload-time = "2025-11-23T14:16:28.472Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -375,6 +450,48 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, ] +[[package]] +name = "fire" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "termcolor" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/00/f8d10588d2019d6d6452653def1ee807353b21983db48550318424b5ff18/fire-0.7.1.tar.gz", hash = "sha256:3b208f05c736de98fb343310d090dcc4d8c78b2a89ea4f32b837c586270a9cbf", size = 88720, upload-time = "2025-08-16T20:20:24.175Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/4c/93d0f85318da65923e4b91c1c2ff03d8a458cbefebe3bc612a6693c7906d/fire-0.7.1-py3-none-any.whl", hash = "sha256:e43fd8a5033a9001e7e2973bab96070694b9f12f2e0ecf96d4683971b5ab1882", size = 115945, upload-time = "2025-08-16T20:20:22.87Z" }, +] + +[[package]] +name = "flask" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blinker" }, + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "markupsafe" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/6d/cfe3c0fcc5e477df242b98bfe186a4c34357b4847e87ecaef04507332dab/flask-3.1.2.tar.gz", hash = "sha256:bf656c15c80190ed628ad08cdfd3aaa35beb087855e2f494910aa3774cc4fd87", size = 720160, upload-time = "2025-08-19T21:03:21.205Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl", hash = "sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c", size = 103308, upload-time = "2025-08-19T21:03:19.499Z" }, +] + +[[package]] +name = "flask-cors" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flask" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/74/0fc0fa68d62f21daef41017dafab19ef4b36551521260987eb3a5394c7ba/flask_cors-6.0.2.tar.gz", hash = "sha256:6e118f3698249ae33e429760db98ce032a8bf9913638d085ca0f4c5534ad2423", size = 13472, upload-time = "2025-12-12T20:31:42.861Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/af/72ad54402e599152de6d067324c46fe6a4f531c7c65baf7e96c63db55eaf/flask_cors-6.0.2-py3-none-any.whl", hash = "sha256:e57544d415dfd7da89a9564e1e3a9e515042df76e12130641ca6f3f2f03b699a", size = 13257, upload-time = "2025-12-12T20:31:41.3Z" }, +] + [[package]] name = "flatbuffers" version = "25.9.23" @@ -393,6 +510,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" }, ] +[[package]] +name = "gast" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/f6/e73969782a2ecec280f8a176f2476149dd9dba69d5f8779ec6108a7721e6/gast-0.7.0.tar.gz", hash = "sha256:0bb14cd1b806722e91ddbab6fb86bba148c22b40e7ff11e248974e04c8adfdae", size = 33630, upload-time = "2025-11-29T15:30:05.266Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/33/f1c6a276de27b7d7339a34749cc33fa87f077f921969c47185d34a887ae2/gast-0.7.0-py3-none-any.whl", hash = "sha256:99cbf1365633a74099f69c59bd650476b96baa5ef196fec88032b00b31ba36f7", size = 22966, upload-time = "2025-11-29T15:30:03.983Z" }, +] + +[[package]] +name = "gdown" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "filelock" }, + { name = "requests", extra = ["socks"] }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/cf/919a9fa16faf8e4572a24d941353edaf4d54e3ddcd048e6c1aeb8c7a9903/gdown-5.2.1.tar.gz", hash = "sha256:247c2ad1f579db5b66b54c04e6a871995fc8fd7021708b950b8ba7b32cf90323", size = 284743, upload-time = "2026-01-11T09:34:01.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/21/35dd0a0b7428bd67b12b358d7b4277f693493a3839b071d540a4c8357b78/gdown-5.2.1-py3-none-any.whl", hash = "sha256:391f0480d495fb87644d1a1ee3ddfeb2144e1de31408fbc74f7e3b3ba927052b", size = 18241, upload-time = "2026-01-11T09:34:02.637Z" }, +] + +[[package]] +name = "google-pasta" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/4a/0bd53b36ff0323d10d5f24ebd67af2de10a1117f5cf4d7add90df92756f1/google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e", size = 40430, upload-time = "2020-03-13T18:57:50.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed", size = 57471, upload-time = "2020-03-13T18:57:48.872Z" }, +] + +[[package]] +name = "grpcio" +version = "1.76.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/e0/318c1ce3ae5a17894d5791e87aea147587c9e702f24122cc7a5c8bbaeeb1/grpcio-1.76.0.tar.gz", hash = "sha256:7be78388d6da1a25c0d5ec506523db58b18be22d9c37d8d3a32c08be4987bd73", size = 12785182, upload-time = "2025-10-21T16:23:12.106Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/ed/71467ab770effc9e8cef5f2e7388beb2be26ed642d567697bb103a790c72/grpcio-1.76.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:26ef06c73eb53267c2b319f43e6634c7556ea37672029241a056629af27c10e2", size = 5807716, upload-time = "2025-10-21T16:21:48.475Z" }, + { url = "https://files.pythonhosted.org/packages/2c/85/c6ed56f9817fab03fa8a111ca91469941fb514e3e3ce6d793cb8f1e1347b/grpcio-1.76.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:45e0111e73f43f735d70786557dc38141185072d7ff8dc1829d6a77ac1471468", size = 11821522, upload-time = "2025-10-21T16:21:51.142Z" }, + { url = "https://files.pythonhosted.org/packages/ac/31/2b8a235ab40c39cbc141ef647f8a6eb7b0028f023015a4842933bc0d6831/grpcio-1.76.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:83d57312a58dcfe2a3a0f9d1389b299438909a02db60e2f2ea2ae2d8034909d3", size = 6362558, upload-time = "2025-10-21T16:21:54.213Z" }, + { url = "https://files.pythonhosted.org/packages/bd/64/9784eab483358e08847498ee56faf8ff6ea8e0a4592568d9f68edc97e9e9/grpcio-1.76.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3e2a27c89eb9ac3d81ec8835e12414d73536c6e620355d65102503064a4ed6eb", size = 7049990, upload-time = "2025-10-21T16:21:56.476Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/8c12319a6369434e7a184b987e8e9f3b49a114c489b8315f029e24de4837/grpcio-1.76.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61f69297cba3950a524f61c7c8ee12e55c486cb5f7db47ff9dcee33da6f0d3ae", size = 6575387, upload-time = "2025-10-21T16:21:59.051Z" }, + { url = "https://files.pythonhosted.org/packages/15/0f/f12c32b03f731f4a6242f771f63039df182c8b8e2cf8075b245b409259d4/grpcio-1.76.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a15c17af8839b6801d554263c546c69c4d7718ad4321e3166175b37eaacca77", size = 7166668, upload-time = "2025-10-21T16:22:02.049Z" }, + { url = "https://files.pythonhosted.org/packages/ff/2d/3ec9ce0c2b1d92dd59d1c3264aaec9f0f7c817d6e8ac683b97198a36ed5a/grpcio-1.76.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:25a18e9810fbc7e7f03ec2516addc116a957f8cbb8cbc95ccc80faa072743d03", size = 8124928, upload-time = "2025-10-21T16:22:04.984Z" }, + { url = "https://files.pythonhosted.org/packages/1a/74/fd3317be5672f4856bcdd1a9e7b5e17554692d3db9a3b273879dc02d657d/grpcio-1.76.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:931091142fd8cc14edccc0845a79248bc155425eee9a98b2db2ea4f00a235a42", size = 7589983, upload-time = "2025-10-21T16:22:07.881Z" }, + { url = "https://files.pythonhosted.org/packages/45/bb/ca038cf420f405971f19821c8c15bcbc875505f6ffadafe9ffd77871dc4c/grpcio-1.76.0-cp313-cp313-win32.whl", hash = "sha256:5e8571632780e08526f118f74170ad8d50fb0a48c23a746bef2a6ebade3abd6f", size = 3984727, upload-time = "2025-10-21T16:22:10.032Z" }, + { url = "https://files.pythonhosted.org/packages/41/80/84087dc56437ced7cdd4b13d7875e7439a52a261e3ab4e06488ba6173b0a/grpcio-1.76.0-cp313-cp313-win_amd64.whl", hash = "sha256:f9f7bd5faab55f47231ad8dba7787866b69f5e93bc306e3915606779bbfb4ba8", size = 4702799, upload-time = "2025-10-21T16:22:12.709Z" }, + { url = "https://files.pythonhosted.org/packages/b4/46/39adac80de49d678e6e073b70204091e76631e03e94928b9ea4ecf0f6e0e/grpcio-1.76.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:ff8a59ea85a1f2191a0ffcc61298c571bc566332f82e5f5be1b83c9d8e668a62", size = 5808417, upload-time = "2025-10-21T16:22:15.02Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f5/a4531f7fb8b4e2a60b94e39d5d924469b7a6988176b3422487be61fe2998/grpcio-1.76.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:06c3d6b076e7b593905d04fdba6a0525711b3466f43b3400266f04ff735de0cd", size = 11828219, upload-time = "2025-10-21T16:22:17.954Z" }, + { url = "https://files.pythonhosted.org/packages/4b/1c/de55d868ed7a8bd6acc6b1d6ddc4aa36d07a9f31d33c912c804adb1b971b/grpcio-1.76.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd5ef5932f6475c436c4a55e4336ebbe47bd3272be04964a03d316bbf4afbcbc", size = 6367826, upload-time = "2025-10-21T16:22:20.721Z" }, + { url = "https://files.pythonhosted.org/packages/59/64/99e44c02b5adb0ad13ab3adc89cb33cb54bfa90c74770f2607eea629b86f/grpcio-1.76.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b331680e46239e090f5b3cead313cc772f6caa7d0fc8de349337563125361a4a", size = 7049550, upload-time = "2025-10-21T16:22:23.637Z" }, + { url = "https://files.pythonhosted.org/packages/43/28/40a5be3f9a86949b83e7d6a2ad6011d993cbe9b6bd27bea881f61c7788b6/grpcio-1.76.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2229ae655ec4e8999599469559e97630185fdd53ae1e8997d147b7c9b2b72cba", size = 6575564, upload-time = "2025-10-21T16:22:26.016Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a9/1be18e6055b64467440208a8559afac243c66a8b904213af6f392dc2212f/grpcio-1.76.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:490fa6d203992c47c7b9e4a9d39003a0c2bcc1c9aa3c058730884bbbb0ee9f09", size = 7176236, upload-time = "2025-10-21T16:22:28.362Z" }, + { url = "https://files.pythonhosted.org/packages/0f/55/dba05d3fcc151ce6e81327541d2cc8394f442f6b350fead67401661bf041/grpcio-1.76.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:479496325ce554792dba6548fae3df31a72cef7bad71ca2e12b0e58f9b336bfc", size = 8125795, upload-time = "2025-10-21T16:22:31.075Z" }, + { url = "https://files.pythonhosted.org/packages/4a/45/122df922d05655f63930cf42c9e3f72ba20aadb26c100ee105cad4ce4257/grpcio-1.76.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c9b93f79f48b03ada57ea24725d83a30284a012ec27eab2cf7e50a550cbbbcc", size = 7592214, upload-time = "2025-10-21T16:22:33.831Z" }, + { url = "https://files.pythonhosted.org/packages/4a/6e/0b899b7f6b66e5af39e377055fb4a6675c9ee28431df5708139df2e93233/grpcio-1.76.0-cp314-cp314-win32.whl", hash = "sha256:747fa73efa9b8b1488a95d0ba1039c8e2dca0f741612d80415b1e1c560febf4e", size = 4062961, upload-time = "2025-10-21T16:22:36.468Z" }, + { url = "https://files.pythonhosted.org/packages/19/41/0b430b01a2eb38ee887f88c1f07644a1df8e289353b78e82b37ef988fb64/grpcio-1.76.0-cp314-cp314-win_amd64.whl", hash = "sha256:922fa70ba549fce362d2e2871ab542082d66e2aaf0c19480ea453905b01f384e", size = 4834462, upload-time = "2025-10-21T16:22:39.772Z" }, +] + +[[package]] +name = "gunicorn" +version = "23.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -402,6 +598,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h5py" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/6a/0d79de0b025aa85dc8864de8e97659c94cf3d23148394a954dc5ca52f8c8/h5py-3.15.1.tar.gz", hash = "sha256:c86e3ed45c4473564de55aa83b6fc9e5ead86578773dfbd93047380042e26b69", size = 426236, upload-time = "2025-10-16T10:35:27.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b3/40207e0192415cbff7ea1d37b9f24b33f6d38a5a2f5d18a678de78f967ae/h5py-3.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c8440fd8bee9500c235ecb7aa1917a0389a2adb80c209fa1cc485bd70e0d94a5", size = 3376511, upload-time = "2025-10-16T10:34:38.596Z" }, + { url = "https://files.pythonhosted.org/packages/31/96/ba99a003c763998035b0de4c299598125df5fc6c9ccf834f152ddd60e0fb/h5py-3.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ab2219dbc6fcdb6932f76b548e2b16f34a1f52b7666e998157a4dfc02e2c4123", size = 2826143, upload-time = "2025-10-16T10:34:41.342Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c2/fc6375d07ea3962df7afad7d863fe4bde18bb88530678c20d4c90c18de1d/h5py-3.15.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8cb02c3a96255149ed3ac811eeea25b655d959c6dd5ce702c9a95ff11859eb5", size = 4908316, upload-time = "2025-10-16T10:34:44.619Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/4402ea66272dacc10b298cca18ed73e1c0791ff2ae9ed218d3859f9698ac/h5py-3.15.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:121b2b7a4c1915d63737483b7bff14ef253020f617c2fb2811f67a4bed9ac5e8", size = 5103710, upload-time = "2025-10-16T10:34:48.639Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f6/11f1e2432d57d71322c02a97a5567829a75f223a8c821764a0e71a65cde8/h5py-3.15.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59b0d63b318bf3cc06687def2b45afd75926bbc006f7b8cd2b1a231299fc8599", size = 4556042, upload-time = "2025-10-16T10:34:51.841Z" }, + { url = "https://files.pythonhosted.org/packages/18/88/3eda3ef16bfe7a7dbc3d8d6836bbaa7986feb5ff091395e140dc13927bcc/h5py-3.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e02fe77a03f652500d8bff288cbf3675f742fc0411f5a628fa37116507dc7cc0", size = 5030639, upload-time = "2025-10-16T10:34:55.257Z" }, + { url = "https://files.pythonhosted.org/packages/e5/ea/fbb258a98863f99befb10ed727152b4ae659f322e1d9c0576f8a62754e81/h5py-3.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:dea78b092fd80a083563ed79a3171258d4a4d307492e7cf8b2313d464c82ba52", size = 2864363, upload-time = "2025-10-16T10:34:58.099Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c9/35021cc9cd2b2915a7da3026e3d77a05bed1144a414ff840953b33937fb9/h5py-3.15.1-cp313-cp313-win_arm64.whl", hash = "sha256:c256254a8a81e2bddc0d376e23e2a6d2dc8a1e8a2261835ed8c1281a0744cd97", size = 2449570, upload-time = "2025-10-16T10:35:00.473Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2c/926eba1514e4d2e47d0e9eb16c784e717d8b066398ccfca9b283917b1bfb/h5py-3.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5f4fb0567eb8517c3ecd6b3c02c4f4e9da220c8932604960fd04e24ee1254763", size = 3380368, upload-time = "2025-10-16T10:35:03.117Z" }, + { url = "https://files.pythonhosted.org/packages/65/4b/d715ed454d3baa5f6ae1d30b7eca4c7a1c1084f6a2edead9e801a1541d62/h5py-3.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:954e480433e82d3872503104f9b285d369048c3a788b2b1a00e53d1c47c98dd2", size = 2833793, upload-time = "2025-10-16T10:35:05.623Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d4/ef386c28e4579314610a8bffebbee3b69295b0237bc967340b7c653c6c10/h5py-3.15.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fd125c131889ebbef0849f4a0e29cf363b48aba42f228d08b4079913b576bb3a", size = 4903199, upload-time = "2025-10-16T10:35:08.972Z" }, + { url = "https://files.pythonhosted.org/packages/33/5d/65c619e195e0b5e54ea5a95c1bb600c8ff8715e0d09676e4cce56d89f492/h5py-3.15.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28a20e1a4082a479b3d7db2169f3a5034af010b90842e75ebbf2e9e49eb4183e", size = 5097224, upload-time = "2025-10-16T10:35:12.808Z" }, + { url = "https://files.pythonhosted.org/packages/30/30/5273218400bf2da01609e1292f562c94b461fcb73c7a9e27fdadd43abc0a/h5py-3.15.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa8df5267f545b4946df8ca0d93d23382191018e4cda2deda4c2cedf9a010e13", size = 4551207, upload-time = "2025-10-16T10:35:16.24Z" }, + { url = "https://files.pythonhosted.org/packages/d3/39/a7ef948ddf4d1c556b0b2b9559534777bccc318543b3f5a1efdf6b556c9c/h5py-3.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99d374a21f7321a4c6ab327c4ab23bd925ad69821aeb53a1e75dd809d19f67fa", size = 5025426, upload-time = "2025-10-16T10:35:19.831Z" }, + { url = "https://files.pythonhosted.org/packages/b6/d8/7368679b8df6925b8415f9dcc9ab1dab01ddc384d2b2c24aac9191bd9ceb/h5py-3.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:9c73d1d7cdb97d5b17ae385153472ce118bed607e43be11e9a9deefaa54e0734", size = 2865704, upload-time = "2025-10-16T10:35:22.658Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b7/4a806f85d62c20157e62e58e03b27513dc9c55499768530acc4f4c5ce4be/h5py-3.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:a6d8c5a05a76aca9a494b4c53ce8a9c29023b7f64f625c6ce1841e92a362ccdf", size = 2465544, upload-time = "2025-10-16T10:35:25.695Z" }, +] + [[package]] name = "hf-xet" version = "1.1.10" @@ -460,14 +683,14 @@ name = "huggingface-hub" version = "0.35.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform == 'darwin'" }, + { name = "fsspec", marker = "sys_platform == 'darwin'" }, + { name = "hf-xet", marker = "(platform_machine == 'aarch64' and sys_platform == 'darwin') or (platform_machine == 'amd64' and sys_platform == 'darwin') or (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')" }, + { name = "packaging", marker = "sys_platform == 'darwin'" }, + { name = "pyyaml", marker = "sys_platform == 'darwin'" }, + { name = "requests", marker = "sys_platform == 'darwin'" }, + { name = "tqdm", marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798, upload-time = "2025-09-29T14:29:58.625Z" } wheels = [ @@ -533,14 +756,85 @@ wheels = [ [[package]] name = "jinja2" -version = "3.0.3" +version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markupsafe" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/91/a5/429efc6246119e1e3fbf562c00187d04e83e54619249eb732bb423efa6c6/Jinja2-3.0.3.tar.gz", hash = "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7", size = 269196, upload-time = "2021-11-09T20:27:29.541Z" } +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/9a/e5d9ec41927401e41aea8af6d16e78b5e612bca4699d417f646a9610a076/Jinja2-3.0.3-py3-none-any.whl", hash = "sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8", size = 133630, upload-time = "2021-11-09T20:27:27.116Z" }, + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + +[[package]] +name = "keras" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "h5py" }, + { name = "ml-dtypes" }, + { name = "namex" }, + { name = "numpy" }, + { name = "optree" }, + { name = "packaging" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/eb/960ec65476a6f5c9223fac7e6ae09999cd75a75fb87afe130e4b1d43f0ac/keras-3.13.0.tar.gz", hash = "sha256:ec51ad2ffcef086d0e3077ac461fa9e3bc54f91d94b49b7c9a84c9af7f54cf5e", size = 1153648, upload-time = "2025-12-17T23:49:25.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/d2/c6734cbf15288d75722ed3eb9d8ebf9204e48379c08160fd40fcd58a0c8b/keras-3.13.0-py3-none-any.whl", hash = "sha256:096793e2be6230816f3f7e030370e66c0f4a89707c59bf2d8fad3ca33869bd1c", size = 1512339, upload-time = "2025-12-17T23:49:24.46Z" }, +] + +[[package]] +name = "libclang" +version = "18.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250", size = 39612, upload-time = "2024-03-17T16:04:37.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/49/f5e3e7e1419872b69f6f5e82ba56e33955a74bd537d8a1f5f1eff2f3668a/libclang-18.1.1-1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:0b2e143f0fac830156feb56f9231ff8338c20aecfe72b4ffe96f19e5a1dbb69a", size = 25836045, upload-time = "2024-06-30T17:40:31.646Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e5/fc61bbded91a8830ccce94c5294ecd6e88e496cc85f6704bf350c0634b70/libclang-18.1.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6f14c3f194704e5d09769108f03185fce7acaf1d1ae4bbb2f30a72c2400cb7c5", size = 26502641, upload-time = "2024-03-18T15:52:26.722Z" }, + { url = "https://files.pythonhosted.org/packages/db/ed/1df62b44db2583375f6a8a5e2ca5432bbdc3edb477942b9b7c848c720055/libclang-18.1.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:83ce5045d101b669ac38e6da8e58765f12da2d3aafb3b9b98d88b286a60964d8", size = 26420207, upload-time = "2024-03-17T15:00:26.63Z" }, + { url = "https://files.pythonhosted.org/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b", size = 24515943, upload-time = "2024-03-17T16:03:45.942Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592", size = 23784972, upload-time = "2024-03-17T16:12:47.677Z" }, + { url = "https://files.pythonhosted.org/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe", size = 20259606, upload-time = "2024-03-17T16:17:42.437Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f", size = 24921494, upload-time = "2024-03-17T16:14:20.132Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2d/3f480b1e1d31eb3d6de5e3ef641954e5c67430d5ac93b7fa7e07589576c7/libclang-18.1.1-py2.py3-none-win_amd64.whl", hash = "sha256:4dd2d3b82fab35e2bf9ca717d7b63ac990a3519c7e312f19fa8e86dcc712f7fb", size = 26415083, upload-time = "2024-03-17T16:42:21.703Z" }, + { url = "https://files.pythonhosted.org/packages/71/cf/e01dc4cc79779cd82d77888a88ae2fa424d93b445ad4f6c02bfc18335b70/libclang-18.1.1-py2.py3-none-win_arm64.whl", hash = "sha256:3f0e1f49f04d3cd198985fea0511576b0aee16f9ff0e0f0cad7f9c57ec3c20e8", size = 22361112, upload-time = "2024-03-17T16:42:59.565Z" }, +] + +[[package]] +name = "lightecc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/ce/d3e427b0b5900248b07ade445f8def3beb810f866f6316aeaa8eae794dad/lightecc-0.0.4.tar.gz", hash = "sha256:ca702aa6891b7a5d9c5bf73c0e9b6e6bbccec2a5b94df7440b99fd4f8bd08050", size = 41636, upload-time = "2025-12-17T19:05:22.317Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/8a/cabe9816cf5032a770b89cd1176d5e2916c2dec69153f25661cc2a8620a6/lightecc-0.0.4-py3-none-any.whl", hash = "sha256:ebc7c2a03548bea44467994a94605f7c5b7153a60cedd27c230ac76380a2fdf3", size = 45282, upload-time = "2025-12-17T19:05:20.949Z" }, +] + +[[package]] +name = "lightphe" +version = "0.0.20" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lightecc" }, + { name = "sympy" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/bb/c9d7bf9c5469bc082a3f3727253a83838b1ffe3c9a926ae01d0f320a0c40/lightphe-0.0.20.tar.gz", hash = "sha256:1d972918ea04630d14c4945a7ed726f572ec8435e1b959f4dc90fc62d2a64e7c", size = 39592, upload-time = "2025-12-17T19:07:25.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/6c/e0f98e23d66c2a0baefcf9e6b8dea90636d34de973fc67f2671f3deb1789/lightphe-0.0.20-py3-none-any.whl", hash = "sha256:a43610371c0586fa5c9d08482f518c9e2fca98c65e7aae0d321d82f63ebedded", size = 59360, upload-time = "2025-12-17T19:07:24.279Z" }, ] [[package]] @@ -556,6 +850,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/09/56/ed35668130e32dbfad2eb37356793b0a95f23494ab5be7d9bf5cb75850ee/llvmlite-0.45.1-cp313-cp313-win_amd64.whl", hash = "sha256:080e6f8d0778a8239cd47686d402cb66eb165e421efa9391366a9b7e5810a38b", size = 38132232, upload-time = "2025-10-01T18:05:14.477Z" }, ] +[[package]] +name = "lz4" +version = "4.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/46/08fd8ef19b782f301d56a9ccfd7dafec5fd4fc1a9f017cf22a1accb585d7/lz4-4.4.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6bb05416444fafea170b07181bc70640975ecc2a8c92b3b658c554119519716c", size = 207171, upload-time = "2025-11-03T13:01:56.595Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3f/ea3334e59de30871d773963997ecdba96c4584c5f8007fd83cfc8f1ee935/lz4-4.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b424df1076e40d4e884cfcc4c77d815368b7fb9ebcd7e634f937725cd9a8a72a", size = 207163, upload-time = "2025-11-03T13:01:57.721Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/7b3a2a0feb998969f4793c650bb16eff5b06e80d1f7bff867feb332f2af2/lz4-4.4.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:216ca0c6c90719731c64f41cfbd6f27a736d7e50a10b70fad2a9c9b262ec923d", size = 1292136, upload-time = "2025-11-03T13:02:00.375Z" }, + { url = "https://files.pythonhosted.org/packages/89/d1/f1d259352227bb1c185288dd694121ea303e43404aa77560b879c90e7073/lz4-4.4.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:533298d208b58b651662dd972f52d807d48915176e5b032fb4f8c3b6f5fe535c", size = 1279639, upload-time = "2025-11-03T13:02:01.649Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fb/ba9256c48266a09012ed1d9b0253b9aa4fe9cdff094f8febf5b26a4aa2a2/lz4-4.4.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:451039b609b9a88a934800b5fc6ee401c89ad9c175abf2f4d9f8b2e4ef1afc64", size = 1368257, upload-time = "2025-11-03T13:02:03.35Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6d/dee32a9430c8b0e01bbb4537573cabd00555827f1a0a42d4e24ca803935c/lz4-4.4.5-cp313-cp313-win32.whl", hash = "sha256:a5f197ffa6fc0e93207b0af71b302e0a2f6f29982e5de0fbda61606dd3a55832", size = 88191, upload-time = "2025-11-03T13:02:04.406Z" }, + { url = "https://files.pythonhosted.org/packages/18/e0/f06028aea741bbecb2a7e9648f4643235279a770c7ffaf70bd4860c73661/lz4-4.4.5-cp313-cp313-win_amd64.whl", hash = "sha256:da68497f78953017deb20edff0dba95641cc86e7423dfadf7c0264e1ac60dc22", size = 99502, upload-time = "2025-11-03T13:02:05.886Z" }, + { url = "https://files.pythonhosted.org/packages/61/72/5bef44afb303e56078676b9f2486f13173a3c1e7f17eaac1793538174817/lz4-4.4.5-cp313-cp313-win_arm64.whl", hash = "sha256:c1cfa663468a189dab510ab231aad030970593f997746d7a324d40104db0d0a9", size = 91285, upload-time = "2025-11-03T13:02:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/6a5c2952971af73f15ed4ebfdd69774b454bd0dc905b289082ca8664fba1/lz4-4.4.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67531da3b62f49c939e09d56492baf397175ff39926d0bd5bd2d191ac2bff95f", size = 207348, upload-time = "2025-11-03T13:02:08.117Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d7/fd62cbdbdccc35341e83aabdb3f6d5c19be2687d0a4eaf6457ddf53bba64/lz4-4.4.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a1acbbba9edbcbb982bc2cac5e7108f0f553aebac1040fbec67a011a45afa1ba", size = 207340, upload-time = "2025-11-03T13:02:09.152Z" }, + { url = "https://files.pythonhosted.org/packages/77/69/225ffadaacb4b0e0eb5fd263541edd938f16cd21fe1eae3cd6d5b6a259dc/lz4-4.4.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a482eecc0b7829c89b498fda883dbd50e98153a116de612ee7c111c8bcf82d1d", size = 1293398, upload-time = "2025-11-03T13:02:10.272Z" }, + { url = "https://files.pythonhosted.org/packages/c6/9e/2ce59ba4a21ea5dc43460cba6f34584e187328019abc0e66698f2b66c881/lz4-4.4.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e099ddfaa88f59dd8d36c8a3c66bd982b4984edf127eb18e30bb49bdba68ce67", size = 1281209, upload-time = "2025-11-03T13:02:12.091Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/4d946bd1624ec229b386a3bc8e7a85fa9a963d67d0a62043f0af0978d3da/lz4-4.4.5-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2af2897333b421360fdcce895c6f6281dc3fab018d19d341cf64d043fc8d90d", size = 1369406, upload-time = "2025-11-03T13:02:13.683Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/d429ba4720a9064722698b4b754fb93e42e625f1318b8fe834086c7c783b/lz4-4.4.5-cp313-cp313t-win32.whl", hash = "sha256:66c5de72bf4988e1b284ebdd6524c4bead2c507a2d7f172201572bac6f593901", size = 88325, upload-time = "2025-11-03T13:02:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/7ba10c9b97c06af6c8f7032ec942ff127558863df52d866019ce9d2425cf/lz4-4.4.5-cp313-cp313t-win_amd64.whl", hash = "sha256:cdd4bdcbaf35056086d910d219106f6a04e1ab0daa40ec0eeef1626c27d0fddb", size = 99643, upload-time = "2025-11-03T13:02:15.978Z" }, + { url = "https://files.pythonhosted.org/packages/77/4d/a175459fb29f909e13e57c8f475181ad8085d8d7869bd8ad99033e3ee5fa/lz4-4.4.5-cp313-cp313t-win_arm64.whl", hash = "sha256:28ccaeb7c5222454cd5f60fcd152564205bcb801bd80e125949d2dfbadc76bbd", size = 91504, upload-time = "2025-11-03T13:02:17.313Z" }, + { url = "https://files.pythonhosted.org/packages/63/9c/70bdbdb9f54053a308b200b4678afd13efd0eafb6ddcbb7f00077213c2e5/lz4-4.4.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c216b6d5275fc060c6280936bb3bb0e0be6126afb08abccde27eed23dead135f", size = 207586, upload-time = "2025-11-03T13:02:18.263Z" }, + { url = "https://files.pythonhosted.org/packages/b6/cb/bfead8f437741ce51e14b3c7d404e3a1f6b409c440bad9b8f3945d4c40a7/lz4-4.4.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c8e71b14938082ebaf78144f3b3917ac715f72d14c076f384a4c062df96f9df6", size = 207161, upload-time = "2025-11-03T13:02:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/e7/18/b192b2ce465dfbeabc4fc957ece7a1d34aded0d95a588862f1c8a86ac448/lz4-4.4.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b5e6abca8df9f9bdc5c3085f33ff32cdc86ed04c65e0355506d46a5ac19b6e9", size = 1292415, upload-time = "2025-11-03T13:02:20.829Z" }, + { url = "https://files.pythonhosted.org/packages/67/79/a4e91872ab60f5e89bfad3e996ea7dc74a30f27253faf95865771225ccba/lz4-4.4.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b84a42da86e8ad8537aabef062e7f661f4a877d1c74d65606c49d835d36d668", size = 1279920, upload-time = "2025-11-03T13:02:22.013Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/d52c7b11eaa286d49dae619c0eec4aabc0bf3cda7a7467eb77c62c4471f3/lz4-4.4.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bba042ec5a61fa77c7e380351a61cb768277801240249841defd2ff0a10742f", size = 1368661, upload-time = "2025-11-03T13:02:23.208Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/137ddeea14c2cb86864838277b2607d09f8253f152156a07f84e11768a28/lz4-4.4.5-cp314-cp314-win32.whl", hash = "sha256:bd85d118316b53ed73956435bee1997bd06cc66dd2fa74073e3b1322bd520a67", size = 90139, upload-time = "2025-11-03T13:02:24.301Z" }, + { url = "https://files.pythonhosted.org/packages/18/2c/8332080fd293f8337779a440b3a143f85e374311705d243439a3349b81ad/lz4-4.4.5-cp314-cp314-win_amd64.whl", hash = "sha256:92159782a4502858a21e0079d77cdcaade23e8a5d252ddf46b0652604300d7be", size = 101497, upload-time = "2025-11-03T13:02:25.187Z" }, + { url = "https://files.pythonhosted.org/packages/ca/28/2635a8141c9a4f4bc23f5135a92bbcf48d928d8ca094088c962df1879d64/lz4-4.4.5-cp314-cp314-win_arm64.whl", hash = "sha256:d994b87abaa7a88ceb7a37c90f547b8284ff9da694e6afcfaa8568d739faf3f7", size = 93812, upload-time = "2025-11-03T13:02:26.133Z" }, +] + +[[package]] +name = "markdown" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/7dd27d9d863b3376fcf23a5a13cb5d024aed1db46f963f1b5735ae43b3be/markdown-3.10.tar.gz", hash = "sha256:37062d4f2aa4b2b6b32aefb80faa300f82cc790cb949a35b8caede34f2b68c0e", size = 364931, upload-time = "2025-11-03T19:51:15.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl", hash = "sha256:b5b99d6951e2e4948d939255596523444c0e677c669700b1d17aa4a8a464cb7c", size = 107678, upload-time = "2025-11-03T19:51:13.887Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -629,6 +964,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "ml-dtypes" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/a1/4008f14bbc616cfb1ac5b39ea485f9c63031c4634ab3f4cf72e7541f816a/ml_dtypes-0.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48", size = 676888, upload-time = "2025-11-17T22:31:56.907Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b7/dff378afc2b0d5a7d6cd9d3209b60474d9819d1189d347521e1688a60a53/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b", size = 5036993, upload-time = "2025-11-17T22:31:58.497Z" }, + { url = "https://files.pythonhosted.org/packages/eb/33/40cd74219417e78b97c47802037cf2d87b91973e18bb968a7da48a96ea44/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d", size = 5010956, upload-time = "2025-11-17T22:31:59.931Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8b/200088c6859d8221454825959df35b5244fa9bdf263fd0249ac5fb75e281/ml_dtypes-0.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328", size = 212224, upload-time = "2025-11-17T22:32:01.349Z" }, + { url = "https://files.pythonhosted.org/packages/8f/75/dfc3775cb36367816e678f69a7843f6f03bd4e2bcd79941e01ea960a068e/ml_dtypes-0.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175", size = 160798, upload-time = "2025-11-17T22:32:02.864Z" }, + { url = "https://files.pythonhosted.org/packages/4f/74/e9ddb35fd1dd43b1106c20ced3f53c2e8e7fc7598c15638e9f80677f81d4/ml_dtypes-0.5.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6", size = 702083, upload-time = "2025-11-17T22:32:04.08Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/667060b0aed1aa63166b22897fdf16dca9eb704e6b4bbf86848d5a181aa7/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d", size = 5354111, upload-time = "2025-11-17T22:32:05.546Z" }, + { url = "https://files.pythonhosted.org/packages/40/49/0f8c498a28c0efa5f5c95a9e374c83ec1385ca41d0e85e7cf40e5d519a21/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298", size = 5366453, upload-time = "2025-11-17T22:32:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/8c/27/12607423d0a9c6bbbcc780ad19f1f6baa2b68b18ce4bddcdc122c4c68dc9/ml_dtypes-0.5.4-cp313-cp313t-win_amd64.whl", hash = "sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6", size = 225612, upload-time = "2025-11-17T22:32:08.615Z" }, + { url = "https://files.pythonhosted.org/packages/e5/80/5a5929e92c72936d5b19872c5fb8fc09327c1da67b3b68c6a13139e77e20/ml_dtypes-0.5.4-cp313-cp313t-win_arm64.whl", hash = "sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1", size = 164145, upload-time = "2025-11-17T22:32:09.782Z" }, + { url = "https://files.pythonhosted.org/packages/72/4e/1339dc6e2557a344f5ba5590872e80346f76f6cb2ac3dd16e4666e88818c/ml_dtypes-0.5.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22", size = 673781, upload-time = "2025-11-17T22:32:11.364Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/067b84365c7e83bda15bba2b06c6ca250ce27b20630b1128c435fb7a09aa/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465", size = 5036145, upload-time = "2025-11-17T22:32:12.783Z" }, + { url = "https://files.pythonhosted.org/packages/c6/bb/82c7dcf38070b46172a517e2334e665c5bf374a262f99a283ea454bece7c/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f", size = 5010230, upload-time = "2025-11-17T22:32:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e9/93/2bfed22d2498c468f6bcd0d9f56b033eaa19f33320389314c19ef6766413/ml_dtypes-0.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56", size = 221032, upload-time = "2025-11-17T22:32:15.763Z" }, + { url = "https://files.pythonhosted.org/packages/76/a3/9c912fe6ea747bb10fe2f8f54d027eb265db05dfb0c6335e3e063e74e6e8/ml_dtypes-0.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049", size = 163353, upload-time = "2025-11-17T22:32:16.932Z" }, + { url = "https://files.pythonhosted.org/packages/cd/02/48aa7d84cc30ab4ee37624a2fd98c56c02326785750cd212bc0826c2f15b/ml_dtypes-0.5.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9", size = 702085, upload-time = "2025-11-17T22:32:18.175Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e7/85cb99fe80a7a5513253ec7faa88a65306be071163485e9a626fce1b6e84/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7", size = 5355358, upload-time = "2025-11-17T22:32:19.7Z" }, + { url = "https://files.pythonhosted.org/packages/79/2b/a826ba18d2179a56e144aef69e57fb2ab7c464ef0b2111940ee8a3a223a2/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf", size = 5366332, upload-time = "2025-11-17T22:32:21.193Z" }, + { url = "https://files.pythonhosted.org/packages/84/44/f4d18446eacb20ea11e82f133ea8f86e2bf2891785b67d9da8d0ab0ef525/ml_dtypes-0.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1", size = 236612, upload-time = "2025-11-17T22:32:22.579Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3f/3d42e9a78fe5edf792a83c074b13b9b770092a4fbf3462872f4303135f09/ml_dtypes-0.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d", size = 168825, upload-time = "2025-11-17T22:32:23.766Z" }, +] + [[package]] name = "mlx" version = "0.29.2" @@ -657,15 +1023,15 @@ name = "mlx-whisper" version = "0.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "mlx" }, - { name = "more-itertools" }, - { name = "numba" }, - { name = "numpy" }, - { name = "scipy" }, - { name = "tiktoken" }, - { name = "torch" }, - { name = "tqdm" }, + { name = "huggingface-hub", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "more-itertools", marker = "sys_platform == 'darwin'" }, + { name = "numba", marker = "sys_platform == 'darwin'" }, + { name = "numpy", marker = "sys_platform == 'darwin'" }, + { name = "scipy", marker = "sys_platform == 'darwin'" }, + { name = "tiktoken", marker = "sys_platform == 'darwin'" }, + { name = "torch", marker = "sys_platform == 'darwin'" }, + { name = "tqdm", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/22/b7/a35232812a2ccfffcb7614ba96a91338551a660a0e9815cee668bf5743f0/mlx_whisper-0.4.3-py3-none-any.whl", hash = "sha256:6b82b6597a994643a3e5496c7bc229a672e5ca308458455bfe276e76ae024489", size = 890544, upload-time = "2025-08-29T14:56:13.815Z" }, @@ -689,6 +1055,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] +[[package]] +name = "mtcnn" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "lz4" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/37/b0f60411b6a37dcd5122bbe05c9aa45f271bcc8129caa45ee1012251905d/mtcnn-1.0.0.tar.gz", hash = "sha256:08428bf8e1ae9827d43a40bb0246b57f2239e3572d3742f472ae9924896c6419", size = 1885746, upload-time = "2024-10-08T01:42:22.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/7e/0b2b688a9e2d353a661b617b12d00d9af29f877b57c8e4a3cbe447483b46/mtcnn-1.0.0-py3-none-any.whl", hash = "sha256:0a96b4868e56db9ae984449519642be6dba03240e608a67e928ebb47833e9144", size = 1898606, upload-time = "2024-10-08T01:42:18.271Z" }, +] + +[[package]] +name = "namex" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0c/c0/ee95b28f029c73f8d49d8f52edaed02a1d4a9acb8b69355737fdb1faa191/namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306", size = 6649, upload-time = "2025-05-26T23:17:38.918Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/bc/465daf1de06409cdd4532082806770ee0d8d7df434da79c76564d0f69741/namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c", size = 5905, upload-time = "2025-05-26T23:17:37.695Z" }, +] + [[package]] name = "networkx" version = "3.5" @@ -813,7 +1201,7 @@ name = "nvidia-cudnn-cu12" version = "9.10.2.21" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, @@ -824,7 +1212,7 @@ name = "nvidia-cufft-cu12" version = "11.3.3.83" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, @@ -851,9 +1239,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.3.90" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, @@ -864,7 +1252,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.8.93" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, @@ -939,6 +1327,95 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/35/8e/d36f8880bcf18ec026a55807d02fe4c7357da9f25aebd92f85178000c0dc/openai_whisper-20250625.tar.gz", hash = "sha256:37a91a3921809d9f44748ffc73c0a55c9f366c85a3ef5c2ae0cc09540432eb96", size = 803191, upload-time = "2025-06-26T01:06:13.34Z" } +[[package]] +name = "opencv-python" +version = "4.11.0.86" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956, upload-time = "2025-01-16T13:52:24.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/4d/53b30a2a3ac1f75f65a59eb29cf2ee7207ce64867db47036ad61743d5a23/opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:432f67c223f1dc2824f5e73cdfcd9db0efc8710647d4e813012195dc9122a52a", size = 37326322, upload-time = "2025-01-16T13:52:25.887Z" }, + { url = "https://files.pythonhosted.org/packages/3b/84/0a67490741867eacdfa37bc18df96e08a9d579583b419010d7f3da8ff503/opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:9d05ef13d23fe97f575153558653e2d6e87103995d54e6a35db3f282fe1f9c66", size = 56723197, upload-time = "2025-01-16T13:55:21.222Z" }, + { url = "https://files.pythonhosted.org/packages/f3/bd/29c126788da65c1fb2b5fb621b7fed0ed5f9122aa22a0868c5e2c15c6d23/opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b92ae2c8852208817e6776ba1ea0d6b1e0a1b5431e971a2a0ddd2a8cc398202", size = 42230439, upload-time = "2025-01-16T13:51:35.822Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8b/90eb44a40476fa0e71e05a0283947cfd74a5d36121a11d926ad6f3193cc4/opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b02611523803495003bd87362db3e1d2a0454a6a63025dc6658a9830570aa0d", size = 62986597, upload-time = "2025-01-16T13:52:08.836Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d7/1d5941a9dde095468b288d989ff6539dd69cd429dbf1b9e839013d21b6f0/opencv_python-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:810549cb2a4aedaa84ad9a1c92fbfdfc14090e2749cedf2c1589ad8359aa169b", size = 29384337, upload-time = "2025-01-16T13:52:13.549Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7d/f1c30a92854540bf789e9cd5dde7ef49bbe63f855b85a2e6b3db8135c591/opencv_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec", size = 39488044, upload-time = "2025-01-16T13:52:21.928Z" }, +] + +[[package]] +name = "opt-einsum" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" }, +] + +[[package]] +name = "optree" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/8e/09d899ad531d50b79aa24e7558f604980fe4048350172e643bb1b9983aec/optree-0.18.0.tar.gz", hash = "sha256:3804fb6ddc923855db2dc4805b4524c66e00f1ef30b166be4aadd52822b13e06", size = 165178, upload-time = "2025-11-14T08:58:31.234Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/60/57874760770dba39e799c88505898b7441786cea24d78bfe0a171e893212/optree-0.18.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:8d88c00c70b5914904feaf8f505f3512c2f3f4493dbbd93951fcdddc85dcfe8c", size = 876547, upload-time = "2025-11-14T08:57:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bb/413263435c557193c436d977689d1c560a08e362f5bca29e3d62b093412a/optree-0.18.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:c8841d44f3648b0662e99fc39ef8c248726ddfb4d1bfce4bdba982e51bb7e3f8", size = 876759, upload-time = "2025-11-14T08:57:13.673Z" }, + { url = "https://files.pythonhosted.org/packages/00/6a/c0f03b83fe888af829591561af398bb7bbe1ea770c7e7475b4d464b4dd7c/optree-0.18.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:385bd727cc7bd3c01bd6204028ac2adce8a8f622c296053d9df434aa0e30b01f", size = 340330, upload-time = "2025-11-14T08:57:14.749Z" }, + { url = "https://files.pythonhosted.org/packages/10/e1/ea857ed58f36c7d2071aef8f67ca0c911e45ded8cb482636185e842550ae/optree-0.18.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6fc9f8acde3bb561b2034e96079507fbe6d4624058fe204161eb8ef29f961296", size = 346098, upload-time = "2025-11-14T08:57:15.862Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7c/9ed10c406028c6b215cd26be4a7afd711a323fd98f531432c1d2921f188b/optree-0.18.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:71ca2fcad8972ba56d6cfffbcd962f45f5d4bc04182f23d66154b38c2eb37de3", size = 372349, upload-time = "2025-11-14T08:57:16.935Z" }, + { url = "https://files.pythonhosted.org/packages/09/67/51acf67b1b9850e990a1a9b3fa0afcb5bbe9d645b0b6b8be5b3f2dca8f04/optree-0.18.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fa8e3878a1857761d64f08a23b32140d29754a53f85f7c87186ced2b5b1b49cb", size = 346522, upload-time = "2025-11-14T08:57:17.953Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/ae957579e22d53d4d24de6bad0a3b3811612fd70a8ecd0c85c81253f22e3/optree-0.18.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27611c6c122745a003b5be7aedba49ef86e9fef46d743c234596de0bde6dc679", size = 368715, upload-time = "2025-11-14T08:57:19.392Z" }, + { url = "https://files.pythonhosted.org/packages/e1/6d/9a3399af22aea044a58e1734257b575b9b17eb67c2c6fcbbb194268e6946/optree-0.18.0-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:cbb083a15ea968ad99e7da17d24632348d69e26534e83c69941f3020ed7536eb", size = 430189, upload-time = "2025-11-14T08:57:20.552Z" }, + { url = "https://files.pythonhosted.org/packages/94/07/9c63a8cad90993848ac6cae5162e2e40f62e9a0738cb522972662ef3c7ab/optree-0.18.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d25941de1acba176305dbdeb931dea6143b30d64ebdc5bfea2bfc12ef9e2b0a", size = 424979, upload-time = "2025-11-14T08:57:21.96Z" }, + { url = "https://files.pythonhosted.org/packages/4a/82/0ab26372377ba1a422a6f38d8237bb2d061dcd23be85bc3ed77404f7b05c/optree-0.18.0-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1db0a6497203a13063a8f044ae751dd5d8253cb815359270c38de0e4c9f8bed5", size = 423201, upload-time = "2025-11-14T08:57:23.047Z" }, + { url = "https://files.pythonhosted.org/packages/19/68/0a761a4f1b2e56ffbf3f223e967074c1331404f6dfb2b2cda6ecf62f4653/optree-0.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:328857d7a35129904b21164f6b0c2ff1d728ad1f5838589c5f437a16c94213c8", size = 414079, upload-time = "2025-11-14T08:57:24.205Z" }, + { url = "https://files.pythonhosted.org/packages/5d/80/29a767bff7413aa593075477a9d17a05d5098bfc0878c087e6b76a3b15df/optree-0.18.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:9d4b9d8c7e9335120ecf222d817699d17de743ad118080fb40467c367f009143", size = 368196, upload-time = "2025-11-14T08:57:25.328Z" }, + { url = "https://files.pythonhosted.org/packages/16/b6/7dfeb866a56f478103faaede5488e55f03916fa707de716ead34dd6f2c3f/optree-0.18.0-cp313-cp313-win32.whl", hash = "sha256:8b9ad4a01a1346b11acc574b7f932dea1a7c7ab31d93546a7540a1f02b3e724a", size = 287207, upload-time = "2025-11-14T08:57:26.461Z" }, + { url = "https://files.pythonhosted.org/packages/7f/92/6e803aa6bf441fae18874f1953e656e179d402b7cbc00c33ae68f0b632db/optree-0.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:5b75e32c191e4b8cf42a8aa854ed264df82936136c0bcad77be44605da41cdfc", size = 314929, upload-time = "2025-11-14T08:57:27.762Z" }, + { url = "https://files.pythonhosted.org/packages/bd/85/e59302d4286552d2694b118e6f5a886490cfd939751c2011b2d3638b2d02/optree-0.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:8a4ca121b6fc6b04300fa225fe6c31897e424db0d92691875af326f8c4e1cead", size = 317459, upload-time = "2025-11-14T08:57:28.826Z" }, + { url = "https://files.pythonhosted.org/packages/44/c9/2009e027f500fb38920d349523dd06b5714687905be24fe06bab90082706/optree-0.18.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:27b1d0cadcf4627c98abbbdce912dbc2243f5687f3c7df39963b793c89321c65", size = 415598, upload-time = "2025-11-14T08:57:29.936Z" }, + { url = "https://files.pythonhosted.org/packages/e2/d1/de1d6d8654d4765a439f27a155d098092ec8670039e2e0ec8383383a2fe7/optree-0.18.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b8adc912ecb6e4fd9df227ded66efaa6702f46a98e1403554be3c9c51d0ca920", size = 387016, upload-time = "2025-11-14T08:57:31.071Z" }, + { url = "https://files.pythonhosted.org/packages/ec/30/6ce07f763b6c0d967a2d683a486eb4450ec053aeae9731133dba600232b2/optree-0.18.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5bc1221068a58175e0ad62afc199893f77c653206673a5552992a604c66fb77e", size = 386289, upload-time = "2025-11-14T08:57:32.146Z" }, + { url = "https://files.pythonhosted.org/packages/76/26/14ed2ff6a69490754446910280a8d0195c489e9fe610d37046b254971627/optree-0.18.0-cp313-cp313t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a479fa25b6e2430e530d00f0c27a55e15ecb9de8ad2d0aec3d40b680e2d6df64", size = 442286, upload-time = "2025-11-14T08:57:33.285Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c7/50bd556ffc76a1cdac1b7460428dee62f8359b60ed07c9846eab0acb5696/optree-0.18.0-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:446c46c53cb8f13abcc0d7dd1989d59bb059953c122fe9901ef53de7fb38b33e", size = 438254, upload-time = "2025-11-14T08:57:34.358Z" }, + { url = "https://files.pythonhosted.org/packages/47/0e/bb9edf64e79f275e5f59fc3dcc49841147cff81598e99e56413523050506/optree-0.18.0-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:81e755124b77e766166c9d05206b90c68f234f425ad2e3c8a6c96f0db548c67b", size = 437817, upload-time = "2025-11-14T08:57:35.382Z" }, + { url = "https://files.pythonhosted.org/packages/be/c4/808b606f840cb53fca2a94cbe82ff26fe23965484dfc4fbb49b6232f990b/optree-0.18.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ae6945f68771b1389ee46a1778e779f4ad76bca9306f3e39eb397f9a0dd2753", size = 426692, upload-time = "2025-11-14T08:57:36.652Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b7/4156ec100d5539400e85ec213e86e154c396efa6135be277de74e19748e2/optree-0.18.0-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:571b732229d7b2e7a2215f57586f8ec0140e07c0faea916e456cbbfa819e56cb", size = 387482, upload-time = "2025-11-14T08:57:37.806Z" }, + { url = "https://files.pythonhosted.org/packages/a0/3b/76a1b45688be72e37965aa467296ebbc743786492287d45907e045933625/optree-0.18.0-cp313-cp313t-win32.whl", hash = "sha256:3014537ff7e4e091ee46e57976f7d95c52f66a0e3eb5ebcbe0de0d924504b58e", size = 318347, upload-time = "2025-11-14T08:57:39.153Z" }, + { url = "https://files.pythonhosted.org/packages/b7/21/87a30a42f1c14365099dc2d656c73bef90a2becbaa1249eca09bf4d9277b/optree-0.18.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a63df296fec376c5cd08298a85109db4a130f4cc8df15916fc92d44ef6068937", size = 351794, upload-time = "2025-11-14T08:57:40.247Z" }, + { url = "https://files.pythonhosted.org/packages/bc/34/37f409de017aa06ee98a01ddb8b93960bd29459f01f090cc461a250977d2/optree-0.18.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9460cba62e941626beb75c99a803373b38a52136d5f1932fcdfdcede1df6f2ef", size = 351225, upload-time = "2025-11-14T08:57:41.582Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f0/dabcb70f2065f782e4c2fac18bde75267d24aa5813b58e7ae9e045ecf9f0/optree-0.18.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:5b126c34b459ef4f10f3a4d7d222416d9102b3c5a76b39f346c611792f144821", size = 876006, upload-time = "2025-11-14T08:57:42.676Z" }, + { url = "https://files.pythonhosted.org/packages/3c/da/6d524879da8892ea8a2562278d0aca06827e7c053015806c5853bb9c3bd8/optree-0.18.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:895f23a4cd8aee2c2464efdad2d9bde28a2aaabee634c96423a933f40e74a67e", size = 876251, upload-time = "2025-11-14T08:57:44.173Z" }, + { url = "https://files.pythonhosted.org/packages/54/f8/588807ec9c21bfec2fcf6b3e4f93abac62cad9bc0b8c0e248f1c30d9c160/optree-0.18.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:db00c604c1ae452f6092293bf230984d4f6cbb3ad905a9991e8cf680fd7d1523", size = 339800, upload-time = "2025-11-14T08:57:45.301Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b9/a4214afaa44ff7d8b2c02ed058b318fcfd73af06daeac45d4845ef26d1b6/optree-0.18.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7172b16e87c87160475275e4bfaa6e4067ccde184d2cca65ba25a402a8ed7758", size = 345613, upload-time = "2025-11-14T08:57:46.362Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cd/31ca853e5f1e9002789de46e5263a3f23d9f9cb9fa490c8bf97fb02076c1/optree-0.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5e669f98b9af9f66144c7ae09912d0367ac3182abe016f67cdd15cb45e13c923", size = 371117, upload-time = "2025-11-14T08:57:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/66/e1/45193039b4432f4142eb978c847cd64533c4db7dc5dcdeb406ceac396961/optree-0.18.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0959bac58631e64e2ac6349cc284b37872c24f353b3d73b4682202a431f07d76", size = 346091, upload-time = "2025-11-14T08:57:48.604Z" }, + { url = "https://files.pythonhosted.org/packages/dc/33/8c6efe13c5cccb464bba868203649888dc875d2010c8a1acec0e9af88e37/optree-0.18.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cde70c97e4cc4e997e8fda2266e40a9bff7679c72ab4af6e15e81748a12882cc", size = 370191, upload-time = "2025-11-14T08:57:49.73Z" }, + { url = "https://files.pythonhosted.org/packages/f0/5e/0fffd06757494e88b3e5699f6df2da301dd9bf19a4f31c197c585dc5001e/optree-0.18.0-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:9104fc8915890e7292e5833fc677e4749607c67aa3cf8884677267078201c2f3", size = 430059, upload-time = "2025-11-14T08:57:50.837Z" }, + { url = "https://files.pythonhosted.org/packages/bf/17/92d0dade6ff46aebad86ae23ac801251e7de18526eee216986de684c3375/optree-0.18.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f674e34202383f8b42fa9335f13bedfb6b6f019c66e1f41034929e4be203423", size = 426169, upload-time = "2025-11-14T08:57:52.211Z" }, + { url = "https://files.pythonhosted.org/packages/e5/36/eedcfcd421801578119ff5fb6731cd50c65f57a729f6f76f8fe6f37d9939/optree-0.18.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b75e083137f361377ff8d70df885ab3a1cf8980e4019e3f311237579adadb64", size = 426153, upload-time = "2025-11-14T08:57:53.591Z" }, + { url = "https://files.pythonhosted.org/packages/63/a6/0bf029f0bdd05f49548644267fc69574a7ca18735010a86d736e7a1ed03c/optree-0.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1f20e8754abe312a701ee00d071ddd8502e9d97ca38fbc56204d14a9ffcb41c", size = 413576, upload-time = "2025-11-14T08:57:54.714Z" }, + { url = "https://files.pythonhosted.org/packages/5e/de/71c51bdf6053e6d7cbdf176eb30d7b5c5ad6180eb6e822d13b36b1edecff/optree-0.18.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:289b184cc41dfc400a30db6207ec997884d14540aae2cba10cb88dc7ebaae2a1", size = 369173, upload-time = "2025-11-14T08:57:56.169Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b0/ea9d562ca87f25bb90944eb411d1ac29ec6c7e38cebf2024e8124fd0e31d/optree-0.18.0-cp314-cp314-win32.whl", hash = "sha256:f5197f864630162f008f5dfad3fceef32553c0fa7639eee1b8e280d924ed678e", size = 292058, upload-time = "2025-11-14T08:57:57.184Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0c/87c9ced927a4cda6b99959cc9341e0a1acb4cd6eb49d2ccd7ac57039c63e/optree-0.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2b5cfb5fc643f16d3a7d957807e55a937dce07566c49ccc4aa71b01064c56758", size = 322019, upload-time = "2025-11-14T08:57:58.203Z" }, + { url = "https://files.pythonhosted.org/packages/39/a8/481afd23d2e66fddc5891b1540778ebedae90e770fe44c68c9f3dbd9e321/optree-0.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:89d5156f8a0a3792701e1c31473eb307f0b45696f48dc51d721f1bfe0c3a950f", size = 324966, upload-time = "2025-11-14T08:57:59.413Z" }, + { url = "https://files.pythonhosted.org/packages/e5/76/7ba344abd30ce4e3c29d50936a2f28341a772bcebec2948be9915f2a3ece/optree-0.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:20536964ba2458f166c1e8ab25951e3fc0a5056b651bd08f16be99bb3ffed54a", size = 415280, upload-time = "2025-11-14T08:58:00.806Z" }, + { url = "https://files.pythonhosted.org/packages/89/27/90de0dcbfdaf82ce616eaa2193a540ec7b4dd1587a5ff0c6a7485c846dd6/optree-0.18.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:07c5f64783ad0f0f80e61c25f276ce79b47deda83ed7956a4a9af6385fe8f60d", size = 387087, upload-time = "2025-11-14T08:58:02.501Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ff/91b9898b08b6f3187a4c99836648893f68d62f61a304b6f6ec61d3e27a77/optree-0.18.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30a2636279bdc805c8e154a0f346bcf704626b831ff44724d305fb72c90b7389", size = 386244, upload-time = "2025-11-14T08:58:03.628Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b2/d20c302926c6c18c379801a6532c0722f7f1a305b7d5712e437708ebdb42/optree-0.18.0-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:80d971060c888c3989132b7e75dfb50848636d41bc931af1b93fe2019fba469c", size = 442297, upload-time = "2025-11-14T08:58:04.887Z" }, + { url = "https://files.pythonhosted.org/packages/90/7d/015c58cf2b0aa0049ac33e1aa76b1fd4563551aeb9f83b10c2217668c355/optree-0.18.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d569730b2647c51a5ee68d67198aa9a78c7a55563d57b8cc1ca8d8c8377e7621", size = 438180, upload-time = "2025-11-14T08:58:06.252Z" }, + { url = "https://files.pythonhosted.org/packages/38/83/1eea2619385bd3ecfda76bb563f4127dc8b4197dcb614eb3f9032c82c2a7/optree-0.18.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c017539e1196ea08f20aea3a4c473f758149b851678edd3d15773b4326decf83", size = 437838, upload-time = "2025-11-14T08:58:07.359Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/c1b84be2143df01819818e8c5db0c284ce995a51134030352ade6d9d1d75/optree-0.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e2cd9ac7fecfd5f6f56ce69f4f805553c226a2744810175959eb408101513c", size = 426705, upload-time = "2025-11-14T08:58:08.426Z" }, + { url = "https://files.pythonhosted.org/packages/70/8a/e1da179a5ebfdb9e279ae655ec38f19e8893a36193203fd6022a31d573b4/optree-0.18.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:a5c213a291c798139ed9ff80aec4bfcd2ac8f001bc015a9cdeb78457e9687dd3", size = 387489, upload-time = "2025-11-14T08:58:09.459Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f8/b1367b93290b9e1b99a5ad1bbedaf76da62cf81578f452a91bfef5cfd1bb/optree-0.18.0-cp314-cp314t-win32.whl", hash = "sha256:e4a468ae1541614b5aa7b4f00254bce005ab7572fbb1fc764af4ee17d90fde7b", size = 327239, upload-time = "2025-11-14T08:58:10.52Z" }, + { url = "https://files.pythonhosted.org/packages/35/84/295aa33e8530c72b45592714a5b07b23e178d2df44baa964c8a91226eac4/optree-0.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:94983b3aa31ee401d2ac77ba570a3157d83f9508cfbb006095a48770e0a1c5ca", size = 366546, upload-time = "2025-11-14T08:58:11.575Z" }, + { url = "https://files.pythonhosted.org/packages/db/67/65af89c4a64b13df70dcf9f09fc42623f490e5b4f4854577679e781c5c32/optree-0.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b4da3223c5b4cf694822752d0fbb6bf34c3f41648af1bd1b443cc3d68cc55106", size = 358524, upload-time = "2025-11-14T08:58:12.967Z" }, +] + [[package]] name = "orjson" version = "3.11.3" @@ -982,6 +1459,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/4b/18b035ee18f97c1040d94debd8f2e737000ad70ccc8f5513f4eefad75f4b/pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713", size = 11544671, upload-time = "2025-09-29T23:21:05.024Z" }, + { url = "https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8", size = 10680807, upload-time = "2025-09-29T23:21:15.979Z" }, + { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872, upload-time = "2025-09-29T23:21:27.165Z" }, + { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371, upload-time = "2025-09-29T23:21:40.532Z" }, + { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333, upload-time = "2025-09-29T23:21:55.77Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120, upload-time = "2025-09-29T23:22:10.109Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c7/e54682c96a895d0c808453269e0b5928a07a127a15704fedb643e9b0a4c8/pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee", size = 10993991, upload-time = "2025-09-29T23:25:04.889Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ca/3f8d4f49740799189e1395812f3bf23b5e8fc7c190827d55a610da72ce55/pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5", size = 12048227, upload-time = "2025-09-29T23:22:24.343Z" }, + { url = "https://files.pythonhosted.org/packages/0e/5a/f43efec3e8c0cc92c4663ccad372dbdff72b60bdb56b2749f04aa1d07d7e/pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21", size = 11411056, upload-time = "2025-09-29T23:22:37.762Z" }, + { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189, upload-time = "2025-09-29T23:22:51.688Z" }, + { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912, upload-time = "2025-09-29T23:23:05.042Z" }, + { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160, upload-time = "2025-09-29T23:23:28.57Z" }, + { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, + { url = "https://files.pythonhosted.org/packages/04/fd/74903979833db8390b73b3a8a7d30d146d710bd32703724dd9083950386f/pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0", size = 11540635, upload-time = "2025-09-29T23:25:52.486Z" }, + { url = "https://files.pythonhosted.org/packages/21/00/266d6b357ad5e6d3ad55093a7e8efc7dd245f5a842b584db9f30b0f0a287/pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593", size = 10759079, upload-time = "2025-09-29T23:26:33.204Z" }, + { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" }, + { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" }, + { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" }, + { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" }, + { url = "https://files.pythonhosted.org/packages/a6/3d/124ac75fcd0ecc09b8fdccb0246ef65e35b012030defb0e0eba2cbbbe948/pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5", size = 11109071, upload-time = "2025-09-29T23:32:27.484Z" }, + { url = "https://files.pythonhosted.org/packages/89/9c/0e21c895c38a157e0faa1fb64587a9226d6dd46452cac4532d80c3c4a244/pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec", size = 12048504, upload-time = "2025-09-29T23:29:31.47Z" }, + { url = "https://files.pythonhosted.org/packages/d7/82/b69a1c95df796858777b68fbe6a81d37443a33319761d7c652ce77797475/pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7", size = 11410702, upload-time = "2025-09-29T23:29:54.591Z" }, + { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" }, + { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" }, + { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" }, +] + [[package]] name = "pepperplus-cb" version = "0.1.0" @@ -989,6 +1506,7 @@ source = { virtual = "." } dependencies = [ { name = "agentspeak" }, { name = "colorlog" }, + { name = "deepface" }, { name = "fastapi", extra = ["all"] }, { name = "mlx-whisper", marker = "sys_platform == 'darwin'" }, { name = "numpy" }, @@ -1038,6 +1556,7 @@ test = [ requires-dist = [ { name = "agentspeak", specifier = ">=0.2.2" }, { name = "colorlog", specifier = ">=6.10.1" }, + { name = "deepface", specifier = ">=0.0.96" }, { name = "fastapi", extras = ["all"], specifier = ">=0.115.6" }, { name = "mlx-whisper", marker = "sys_platform == 'darwin'", specifier = ">=0.4.3" }, { name = "numpy", specifier = ">=2.3.3" }, @@ -1083,6 +1602,64 @@ test = [ { name = "soundfile", specifier = ">=0.13.1" }, ] +[[package]] +name = "pillow" +version = "12.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/02/d52c733a2452ef1ffcc123b68e6606d07276b0e358db70eabad7e40042b7/pillow-12.1.0.tar.gz", hash = "sha256:5c5ae0a06e9ea030ab786b0251b32c7e4ce10e58d983c0d5c56029455180b5b9", size = 46977283, upload-time = "2026-01-02T09:13:29.892Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/c7/2530a4aa28248623e9d7f27316b42e27c32ec410f695929696f2e0e4a778/pillow-12.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7b5dd7cbae20285cdb597b10eb5a2c13aa9de6cde9bb64a3c1317427b1db1ae1", size = 4062543, upload-time = "2026-01-02T09:11:31.566Z" }, + { url = "https://files.pythonhosted.org/packages/8f/1f/40b8eae823dc1519b87d53c30ed9ef085506b05281d313031755c1705f73/pillow-12.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:29a4cef9cb672363926f0470afc516dbf7305a14d8c54f7abbb5c199cd8f8179", size = 4138373, upload-time = "2026-01-02T09:11:33.367Z" }, + { url = "https://files.pythonhosted.org/packages/d4/77/6fa60634cf06e52139fd0e89e5bbf055e8166c691c42fb162818b7fda31d/pillow-12.1.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:681088909d7e8fa9e31b9799aaa59ba5234c58e5e4f1951b4c4d1082a2e980e0", size = 3601241, upload-time = "2026-01-02T09:11:35.011Z" }, + { url = "https://files.pythonhosted.org/packages/4f/bf/28ab865de622e14b747f0cd7877510848252d950e43002e224fb1c9ababf/pillow-12.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:983976c2ab753166dc66d36af6e8ec15bb511e4a25856e2227e5f7e00a160587", size = 5262410, upload-time = "2026-01-02T09:11:36.682Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/583420a1b55e715937a85bd48c5c0991598247a1fd2eb5423188e765ea02/pillow-12.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:db44d5c160a90df2d24a24760bbd37607d53da0b34fb546c4c232af7192298ac", size = 4657312, upload-time = "2026-01-02T09:11:38.535Z" }, + { url = "https://files.pythonhosted.org/packages/1d/fd/f5a0896839762885b3376ff04878f86ab2b097c2f9a9cdccf4eda8ba8dc0/pillow-12.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6b7a9d1db5dad90e2991645874f708e87d9a3c370c243c2d7684d28f7e133e6b", size = 6232605, upload-time = "2026-01-02T09:11:40.602Z" }, + { url = "https://files.pythonhosted.org/packages/98/aa/938a09d127ac1e70e6ed467bd03834350b33ef646b31edb7452d5de43792/pillow-12.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6258f3260986990ba2fa8a874f8b6e808cf5abb51a94015ca3dc3c68aa4f30ea", size = 8041617, upload-time = "2026-01-02T09:11:42.721Z" }, + { url = "https://files.pythonhosted.org/packages/17/e8/538b24cb426ac0186e03f80f78bc8dc7246c667f58b540bdd57c71c9f79d/pillow-12.1.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e115c15e3bc727b1ca3e641a909f77f8ca72a64fff150f666fcc85e57701c26c", size = 6346509, upload-time = "2026-01-02T09:11:44.955Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/632e58ec89a32738cabfd9ec418f0e9898a2b4719afc581f07c04a05e3c9/pillow-12.1.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6741e6f3074a35e47c77b23a4e4f2d90db3ed905cb1c5e6e0d49bff2045632bc", size = 7038117, upload-time = "2026-01-02T09:11:46.736Z" }, + { url = "https://files.pythonhosted.org/packages/c7/a2/d40308cf86eada842ca1f3ffa45d0ca0df7e4ab33c83f81e73f5eaed136d/pillow-12.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:935b9d1aed48fcfb3f838caac506f38e29621b44ccc4f8a64d575cb1b2a88644", size = 6460151, upload-time = "2026-01-02T09:11:48.625Z" }, + { url = "https://files.pythonhosted.org/packages/f1/88/f5b058ad6453a085c5266660a1417bdad590199da1b32fb4efcff9d33b05/pillow-12.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5fee4c04aad8932da9f8f710af2c1a15a83582cfb884152a9caa79d4efcdbf9c", size = 7164534, upload-time = "2026-01-02T09:11:50.445Z" }, + { url = "https://files.pythonhosted.org/packages/19/ce/c17334caea1db789163b5d855a5735e47995b0b5dc8745e9a3605d5f24c0/pillow-12.1.0-cp313-cp313-win32.whl", hash = "sha256:a786bf667724d84aa29b5db1c61b7bfdde380202aaca12c3461afd6b71743171", size = 6332551, upload-time = "2026-01-02T09:11:52.234Z" }, + { url = "https://files.pythonhosted.org/packages/e5/07/74a9d941fa45c90a0d9465098fe1ec85de3e2afbdc15cc4766622d516056/pillow-12.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:461f9dfdafa394c59cd6d818bdfdbab4028b83b02caadaff0ffd433faf4c9a7a", size = 7040087, upload-time = "2026-01-02T09:11:54.822Z" }, + { url = "https://files.pythonhosted.org/packages/88/09/c99950c075a0e9053d8e880595926302575bc742b1b47fe1bbcc8d388d50/pillow-12.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:9212d6b86917a2300669511ed094a9406888362e085f2431a7da985a6b124f45", size = 2452470, upload-time = "2026-01-02T09:11:56.522Z" }, + { url = "https://files.pythonhosted.org/packages/b5/ba/970b7d85ba01f348dee4d65412476321d40ee04dcb51cd3735b9dc94eb58/pillow-12.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:00162e9ca6d22b7c3ee8e61faa3c3253cd19b6a37f126cad04f2f88b306f557d", size = 5264816, upload-time = "2026-01-02T09:11:58.227Z" }, + { url = "https://files.pythonhosted.org/packages/10/60/650f2fb55fdba7a510d836202aa52f0baac633e50ab1cf18415d332188fb/pillow-12.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7d6daa89a00b58c37cb1747ec9fb7ac3bc5ffd5949f5888657dfddde6d1312e0", size = 4660472, upload-time = "2026-01-02T09:12:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c0/5273a99478956a099d533c4f46cbaa19fd69d606624f4334b85e50987a08/pillow-12.1.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2479c7f02f9d505682dc47df8c0ea1fc5e264c4d1629a5d63fe3e2334b89554", size = 6268974, upload-time = "2026-01-02T09:12:02.572Z" }, + { url = "https://files.pythonhosted.org/packages/b4/26/0bf714bc2e73d5267887d47931d53c4ceeceea6978148ed2ab2a4e6463c4/pillow-12.1.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f188d580bd870cda1e15183790d1cc2fa78f666e76077d103edf048eed9c356e", size = 8073070, upload-time = "2026-01-02T09:12:04.75Z" }, + { url = "https://files.pythonhosted.org/packages/43/cf/1ea826200de111a9d65724c54f927f3111dc5ae297f294b370a670c17786/pillow-12.1.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0fde7ec5538ab5095cc02df38ee99b0443ff0e1c847a045554cf5f9af1f4aa82", size = 6380176, upload-time = "2026-01-02T09:12:06.626Z" }, + { url = "https://files.pythonhosted.org/packages/03/e0/7938dd2b2013373fd85d96e0f38d62b7a5a262af21ac274250c7ca7847c9/pillow-12.1.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ed07dca4a8464bada6139ab38f5382f83e5f111698caf3191cb8dbf27d908b4", size = 7067061, upload-time = "2026-01-02T09:12:08.624Z" }, + { url = "https://files.pythonhosted.org/packages/86/ad/a2aa97d37272a929a98437a8c0ac37b3cf012f4f8721e1bd5154699b2518/pillow-12.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f45bd71d1fa5e5749587613037b172e0b3b23159d1c00ef2fc920da6f470e6f0", size = 6491824, upload-time = "2026-01-02T09:12:10.488Z" }, + { url = "https://files.pythonhosted.org/packages/a4/44/80e46611b288d51b115826f136fb3465653c28f491068a72d3da49b54cd4/pillow-12.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:277518bf4fe74aa91489e1b20577473b19ee70fb97c374aa50830b279f25841b", size = 7190911, upload-time = "2026-01-02T09:12:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/86/77/eacc62356b4cf81abe99ff9dbc7402750044aed02cfd6a503f7c6fc11f3e/pillow-12.1.0-cp313-cp313t-win32.whl", hash = "sha256:7315f9137087c4e0ee73a761b163fc9aa3b19f5f606a7fc08d83fd3e4379af65", size = 6336445, upload-time = "2026-01-02T09:12:14.775Z" }, + { url = "https://files.pythonhosted.org/packages/e7/3c/57d81d0b74d218706dafccb87a87ea44262c43eef98eb3b164fd000e0491/pillow-12.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:0ddedfaa8b5f0b4ffbc2fa87b556dc59f6bb4ecb14a53b33f9189713ae8053c0", size = 7045354, upload-time = "2026-01-02T09:12:16.599Z" }, + { url = "https://files.pythonhosted.org/packages/ac/82/8b9b97bba2e3576a340f93b044a3a3a09841170ab4c1eb0d5c93469fd32f/pillow-12.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:80941e6d573197a0c28f394753de529bb436b1ca990ed6e765cf42426abc39f8", size = 2454547, upload-time = "2026-01-02T09:12:18.704Z" }, + { url = "https://files.pythonhosted.org/packages/8c/87/bdf971d8bbcf80a348cc3bacfcb239f5882100fe80534b0ce67a784181d8/pillow-12.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:5cb7bc1966d031aec37ddb9dcf15c2da5b2e9f7cc3ca7c54473a20a927e1eb91", size = 4062533, upload-time = "2026-01-02T09:12:20.791Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4f/5eb37a681c68d605eb7034c004875c81f86ec9ef51f5be4a63eadd58859a/pillow-12.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:97e9993d5ed946aba26baf9c1e8cf18adbab584b99f452ee72f7ee8acb882796", size = 4138546, upload-time = "2026-01-02T09:12:23.664Z" }, + { url = "https://files.pythonhosted.org/packages/11/6d/19a95acb2edbace40dcd582d077b991646b7083c41b98da4ed7555b59733/pillow-12.1.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:414b9a78e14ffeb98128863314e62c3f24b8a86081066625700b7985b3f529bd", size = 3601163, upload-time = "2026-01-02T09:12:26.338Z" }, + { url = "https://files.pythonhosted.org/packages/fc/36/2b8138e51cb42e4cc39c3297713455548be855a50558c3ac2beebdc251dd/pillow-12.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e6bdb408f7c9dd2a5ff2b14a3b0bb6d4deb29fb9961e6eb3ae2031ae9a5cec13", size = 5266086, upload-time = "2026-01-02T09:12:28.782Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/649056e4d22e1caa90816bf99cef0884aed607ed38075bd75f091a607a38/pillow-12.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3413c2ae377550f5487991d444428f1a8ae92784aac79caa8b1e3b89b175f77e", size = 4657344, upload-time = "2026-01-02T09:12:31.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/6b/c5742cea0f1ade0cd61485dc3d81f05261fc2276f537fbdc00802de56779/pillow-12.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e5dcbe95016e88437ecf33544ba5db21ef1b8dd6e1b434a2cb2a3d605299e643", size = 6232114, upload-time = "2026-01-02T09:12:32.936Z" }, + { url = "https://files.pythonhosted.org/packages/bf/8f/9f521268ce22d63991601aafd3d48d5ff7280a246a1ef62d626d67b44064/pillow-12.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d0a7735df32ccbcc98b98a1ac785cc4b19b580be1bdf0aeb5c03223220ea09d5", size = 8042708, upload-time = "2026-01-02T09:12:34.78Z" }, + { url = "https://files.pythonhosted.org/packages/1a/eb/257f38542893f021502a1bbe0c2e883c90b5cff26cc33b1584a841a06d30/pillow-12.1.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c27407a2d1b96774cbc4a7594129cc027339fd800cd081e44497722ea1179de", size = 6347762, upload-time = "2026-01-02T09:12:36.748Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5a/8ba375025701c09b309e8d5163c5a4ce0102fa86bbf8800eb0d7ac87bc51/pillow-12.1.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15c794d74303828eaa957ff8070846d0efe8c630901a1c753fdc63850e19ecd9", size = 7039265, upload-time = "2026-01-02T09:12:39.082Z" }, + { url = "https://files.pythonhosted.org/packages/cf/dc/cf5e4cdb3db533f539e88a7bbf9f190c64ab8a08a9bc7a4ccf55067872e4/pillow-12.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c990547452ee2800d8506c4150280757f88532f3de2a58e3022e9b179107862a", size = 6462341, upload-time = "2026-01-02T09:12:40.946Z" }, + { url = "https://files.pythonhosted.org/packages/d0/47/0291a25ac9550677e22eda48510cfc4fa4b2ef0396448b7fbdc0a6946309/pillow-12.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b63e13dd27da389ed9475b3d28510f0f954bca0041e8e551b2a4eb1eab56a39a", size = 7165395, upload-time = "2026-01-02T09:12:42.706Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4c/e005a59393ec4d9416be06e6b45820403bb946a778e39ecec62f5b2b991e/pillow-12.1.0-cp314-cp314-win32.whl", hash = "sha256:1a949604f73eb07a8adab38c4fe50791f9919344398bdc8ac6b307f755fc7030", size = 6431413, upload-time = "2026-01-02T09:12:44.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/af/f23697f587ac5f9095d67e31b81c95c0249cd461a9798a061ed6709b09b5/pillow-12.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f9f6a650743f0ddee5593ac9e954ba1bdbc5e150bc066586d4f26127853ab94", size = 7176779, upload-time = "2026-01-02T09:12:46.727Z" }, + { url = "https://files.pythonhosted.org/packages/b3/36/6a51abf8599232f3e9afbd16d52829376a68909fe14efe29084445db4b73/pillow-12.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:808b99604f7873c800c4840f55ff389936ef1948e4e87645eaf3fccbc8477ac4", size = 2543105, upload-time = "2026-01-02T09:12:49.243Z" }, + { url = "https://files.pythonhosted.org/packages/82/54/2e1dd20c8749ff225080d6ba465a0cab4387f5db0d1c5fb1439e2d99923f/pillow-12.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bc11908616c8a283cf7d664f77411a5ed2a02009b0097ff8abbba5e79128ccf2", size = 5268571, upload-time = "2026-01-02T09:12:51.11Z" }, + { url = "https://files.pythonhosted.org/packages/57/61/571163a5ef86ec0cf30d265ac2a70ae6fc9e28413d1dc94fa37fae6bda89/pillow-12.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:896866d2d436563fa2a43a9d72f417874f16b5545955c54a64941e87c1376c61", size = 4660426, upload-time = "2026-01-02T09:12:52.865Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e1/53ee5163f794aef1bf84243f755ee6897a92c708505350dd1923f4afec48/pillow-12.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8e178e3e99d3c0ea8fc64b88447f7cac8ccf058af422a6cedc690d0eadd98c51", size = 6269908, upload-time = "2026-01-02T09:12:54.884Z" }, + { url = "https://files.pythonhosted.org/packages/bc/0b/b4b4106ff0ee1afa1dc599fde6ab230417f800279745124f6c50bcffed8e/pillow-12.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:079af2fb0c599c2ec144ba2c02766d1b55498e373b3ac64687e43849fbbef5bc", size = 8074733, upload-time = "2026-01-02T09:12:56.802Z" }, + { url = "https://files.pythonhosted.org/packages/19/9f/80b411cbac4a732439e629a26ad3ef11907a8c7fc5377b7602f04f6fe4e7/pillow-12.1.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdec5e43377761c5dbca620efb69a77f6855c5a379e32ac5b158f54c84212b14", size = 6381431, upload-time = "2026-01-02T09:12:58.823Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b7/d65c45db463b66ecb6abc17c6ba6917a911202a07662247e1355ce1789e7/pillow-12.1.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:565c986f4b45c020f5421a4cea13ef294dde9509a8577f29b2fc5edc7587fff8", size = 7068529, upload-time = "2026-01-02T09:13:00.885Z" }, + { url = "https://files.pythonhosted.org/packages/50/96/dfd4cd726b4a45ae6e3c669fc9e49deb2241312605d33aba50499e9d9bd1/pillow-12.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:43aca0a55ce1eefc0aefa6253661cb54571857b1a7b2964bd8a1e3ef4b729924", size = 6492981, upload-time = "2026-01-02T09:13:03.314Z" }, + { url = "https://files.pythonhosted.org/packages/4d/1c/b5dc52cf713ae46033359c5ca920444f18a6359ce1020dd3e9c553ea5bc6/pillow-12.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0deedf2ea233722476b3a81e8cdfbad786f7adbed5d848469fa59fe52396e4ef", size = 7191878, upload-time = "2026-01-02T09:13:05.276Z" }, + { url = "https://files.pythonhosted.org/packages/53/26/c4188248bd5edaf543864fe4834aebe9c9cb4968b6f573ce014cc42d0720/pillow-12.1.0-cp314-cp314t-win32.whl", hash = "sha256:b17fbdbe01c196e7e159aacb889e091f28e61020a8abeac07b68079b6e626988", size = 6438703, upload-time = "2026-01-02T09:13:07.491Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0e/69ed296de8ea05cb03ee139cee600f424ca166e632567b2d66727f08c7ed/pillow-12.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27b9baecb428899db6c0de572d6d305cfaf38ca1596b5c0542a5182e3e74e8c6", size = 7182927, upload-time = "2026-01-02T09:13:09.841Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f5/68334c015eed9b5cff77814258717dec591ded209ab5b6fb70e2ae873d1d/pillow-12.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f61333d817698bdcdd0f9d7793e365ac3d2a21c1f1eb02b32ad6aefb8d8ea831", size = 2545104, upload-time = "2026-01-02T09:13:12.068Z" }, +] + [[package]] name = "platformdirs" version = "4.5.0" @@ -1260,6 +1837,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178, upload-time = "2024-09-19T02:40:08.598Z" }, ] +[[package]] +name = "pysocks" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, +] + [[package]] name = "pytest" version = "8.4.2" @@ -1314,6 +1900,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + [[package]] name = "python-dotenv" version = "1.1.1" @@ -1341,6 +1939,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" }, ] +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1499,6 +2106,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[package.optional-dependencies] +socks = [ + { name = "pysocks" }, +] + +[[package]] +name = "retina-face" +version = "0.0.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gdown" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "pillow" }, + { name = "tensorflow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/0d/76a74b93c0d9d6fd8ad450986c8f586b371ce74f7845eb3827591d8ac3e1/retina-face-0.0.17.tar.gz", hash = "sha256:7532b136ed01fe9a8cba8dfbc5a046dd6fb1214b1a83e57f3210bd145a91cd73", size = 18929, upload-time = "2024-04-16T21:03:36.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/87/30c5beef6ef3cb60f80f02d3f934b86efda21aca3225f174d127192d43bb/retina_face-0.0.17-py3-none-any.whl", hash = "sha256:b43fdac4078678b9d8bc45b88a7090f05d81c44e1e10710e6c16d703bb7add41", size = 25124, upload-time = "2024-04-16T21:03:35.109Z" }, +] + [[package]] name = "rich" version = "14.1.0" @@ -1629,7 +2257,7 @@ name = "scipy" version = "1.16.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4c/3b/546a6f0bfe791bbb7f8d591613454d15097e53f906308ec6f7c1ce588e8e/scipy-1.16.2.tar.gz", hash = "sha256:af029b153d243a80afb6eabe40b0a07f8e35c9adc269c019f364ad747f826a6b", size = 30580599, upload-time = "2025-09-11T17:48:08.271Z" } wheels = [ @@ -1696,6 +2324,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/6a/a0a024878a1933a2326c42a3ce24fff6c0bf4882655f156c960ba50c2ed4/silero_vad-6.0.0-py3-none-any.whl", hash = "sha256:37d29be8944d2a2e6f1cc38a066076f13e78e6fc1b567a1beddcca72096f077f", size = 6119146, upload-time = "2025-08-26T07:10:00.637Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -1733,6 +2370,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/e9/6b761de83277f2f02ded7e7ea6f07828ec78e4b229b80e4ca55dd205b9dc/soundfile-0.13.1-py2.py3-none-win_amd64.whl", hash = "sha256:1e70a05a0626524a69e9f0f4dd2ec174b4e9567f4d8b6c11d38b5c289be36ee9", size = 1019162, upload-time = "2025-01-25T09:16:59.573Z" }, ] +[[package]] +name = "soupsieve" +version = "2.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/89/23/adf3796d740536d63a6fbda113d07e60c734b6ed5d3058d1e47fc0495e47/soupsieve-2.8.1.tar.gz", hash = "sha256:4cf733bc50fa805f5df4b8ef4740fc0e0fa6218cf3006269afd3f9d6d80fd350", size = 117856, upload-time = "2025-12-18T13:50:34.655Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/f3/b67d6ea49ca9154453b6d70b34ea22f3996b9fa55da105a79d8732227adc/soupsieve-2.8.1-py3-none-any.whl", hash = "sha256:a11fe2a6f3d76ab3cf2de04eb339c1be5b506a8a47f2ceb6d139803177f85434", size = 36710, upload-time = "2025-12-18T13:50:33.267Z" }, +] + [[package]] name = "sphinx" version = "7.3.7" @@ -1864,6 +2510,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] +[[package]] +name = "tensorboard" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "grpcio" }, + { name = "markdown" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "setuptools" }, + { name = "tensorboard-data-server" }, + { name = "werkzeug" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" }, +] + +[[package]] +name = "tensorboard-data-server" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/dabeaf902892922777492e1d253bb7e1264cadce3cea932f7ff599e53fea/tensorboard_data_server-0.7.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:9fe5d24221b29625dbc7328b0436ca7fc1c23de4acf4d272f1180856e32f9f60", size = 4823598, upload-time = "2023-10-23T21:23:33.714Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" }, +] + +[[package]] +name = "tensorflow" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "astunparse" }, + { name = "flatbuffers" }, + { name = "gast" }, + { name = "google-pasta" }, + { name = "grpcio" }, + { name = "h5py" }, + { name = "keras" }, + { name = "libclang" }, + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "opt-einsum" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "requests" }, + { name = "setuptools" }, + { name = "six" }, + { name = "tensorboard" }, + { name = "termcolor" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/82/af283f402f8d1e9315644a331a5f0f326264c5d1de08262f3de5a5ade422/tensorflow-2.20.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:197f0b613b38c0da5c6a12a8295ad4a05c78b853835dae8e0f9dfae3ce9ce8a5", size = 200671458, upload-time = "2025-08-13T16:53:16.568Z" }, + { url = "https://files.pythonhosted.org/packages/ea/4c/c1aa90c5cc92e9f7f9c78421e121ef25bae7d378f8d1d4cbad46c6308836/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47c88e05a07f1ead4977b4894b3ecd4d8075c40191065afc4fd9355c9db3d926", size = 259663776, upload-time = "2025-08-13T16:53:24.507Z" }, + { url = "https://files.pythonhosted.org/packages/43/fb/8be8547c128613d82a2b006004026d86ed0bd672e913029a98153af4ffab/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa3729b0126f75a99882b89fb7d536515721eda8014a63e259e780ba0a37372", size = 620815537, upload-time = "2025-08-13T16:53:42.577Z" }, + { url = "https://files.pythonhosted.org/packages/9b/9e/02e201033f8d6bd5f79240b7262337de44c51a6cfd85c23a86c103c7684d/tensorflow-2.20.0-cp313-cp313-win_amd64.whl", hash = "sha256:c25edad45e8cb9e76366f7a8c835279f9169028d610f3b52ce92d332a1b05438", size = 332012220, upload-time = "2025-08-13T16:53:57.303Z" }, +] + +[[package]] +name = "termcolor" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, +] + [[package]] name = "tiktoken" version = "0.12.0" @@ -1978,7 +2697,7 @@ name = "triton" version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "setuptools" }, + { name = "setuptools", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/30/7b/0a685684ed5322d2af0bddefed7906674f67974aa88b0fae6e82e3b766f6/triton-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00be2964616f4c619193cb0d1b29a99bd4b001d7dc333816073f92cf2a8ccdeb", size = 155569223, upload-time = "2025-07-30T19:58:44.017Z" }, @@ -2021,6 +2740,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "tzdata" +version = "2025.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/a7/c202b344c5ca7daf398f3b8a477eeb205cf3b6f32e7ec3a6bac0629ca975/tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7", size = 196772, upload-time = "2025-12-13T17:45:35.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, +] + [[package]] name = "ujson" version = "5.11.0" @@ -2196,3 +2924,81 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" }, { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] + +[[package]] +name = "werkzeug" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/70/1469ef1d3542ae7c2c7b72bd5e3a4e6ee69d7978fa8a3af05a38eca5becf/werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67", size = 864754, upload-time = "2026-01-08T17:49:23.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/e4/8d97cca767bcc1be76d16fb76951608305561c6e056811587f36cb1316a8/werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", size = 225025, upload-time = "2026-01-08T17:49:21.859Z" }, +] + +[[package]] +name = "wheel" +version = "0.45.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545, upload-time = "2024-11-23T00:18:23.513Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494, upload-time = "2024-11-23T00:18:21.207Z" }, +] + +[[package]] +name = "wrapt" +version = "2.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2a/6de8a50cb435b7f42c46126cf1a54b2aab81784e74c8595c8e025e8f36d3/wrapt-2.0.1.tar.gz", hash = "sha256:9c9c635e78497cacb81e84f8b11b23e0aacac7a136e73b8e5b2109a1d9fc468f", size = 82040, upload-time = "2025-11-07T00:45:33.312Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/fe/41af4c46b5e498c90fc87981ab2972fbd9f0bccda597adb99d3d3441b94b/wrapt-2.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:47b0f8bafe90f7736151f61482c583c86b0693d80f075a58701dd1549b0010a9", size = 78132, upload-time = "2025-11-07T00:44:04.628Z" }, + { url = "https://files.pythonhosted.org/packages/1c/92/d68895a984a5ebbbfb175512b0c0aad872354a4a2484fbd5552e9f275316/wrapt-2.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:cbeb0971e13b4bd81d34169ed57a6dda017328d1a22b62fda45e1d21dd06148f", size = 61211, upload-time = "2025-11-07T00:44:05.626Z" }, + { url = "https://files.pythonhosted.org/packages/e8/26/ba83dc5ae7cf5aa2b02364a3d9cf74374b86169906a1f3ade9a2d03cf21c/wrapt-2.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb7cffe572ad0a141a7886a1d2efa5bef0bf7fe021deeea76b3ab334d2c38218", size = 61689, upload-time = "2025-11-07T00:44:06.719Z" }, + { url = "https://files.pythonhosted.org/packages/cf/67/d7a7c276d874e5d26738c22444d466a3a64ed541f6ef35f740dbd865bab4/wrapt-2.0.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c8d60527d1ecfc131426b10d93ab5d53e08a09c5fa0175f6b21b3252080c70a9", size = 121502, upload-time = "2025-11-07T00:44:09.557Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6b/806dbf6dd9579556aab22fc92908a876636e250f063f71548a8660382184/wrapt-2.0.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c654eafb01afac55246053d67a4b9a984a3567c3808bb7df2f8de1c1caba2e1c", size = 123110, upload-time = "2025-11-07T00:44:10.64Z" }, + { url = "https://files.pythonhosted.org/packages/e5/08/cdbb965fbe4c02c5233d185d070cabed2ecc1f1e47662854f95d77613f57/wrapt-2.0.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:98d873ed6c8b4ee2418f7afce666751854d6d03e3c0ec2a399bb039cd2ae89db", size = 117434, upload-time = "2025-11-07T00:44:08.138Z" }, + { url = "https://files.pythonhosted.org/packages/2d/d1/6aae2ce39db4cb5216302fa2e9577ad74424dfbe315bd6669725569e048c/wrapt-2.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c9e850f5b7fc67af856ff054c71690d54fa940c3ef74209ad9f935b4f66a0233", size = 121533, upload-time = "2025-11-07T00:44:12.142Z" }, + { url = "https://files.pythonhosted.org/packages/79/35/565abf57559fbe0a9155c29879ff43ce8bd28d2ca61033a3a3dd67b70794/wrapt-2.0.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e505629359cb5f751e16e30cf3f91a1d3ddb4552480c205947da415d597f7ac2", size = 116324, upload-time = "2025-11-07T00:44:13.28Z" }, + { url = "https://files.pythonhosted.org/packages/e1/e0/53ff5e76587822ee33e560ad55876d858e384158272cd9947abdd4ad42ca/wrapt-2.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2879af909312d0baf35f08edeea918ee3af7ab57c37fe47cb6a373c9f2749c7b", size = 120627, upload-time = "2025-11-07T00:44:14.431Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7b/38df30fd629fbd7612c407643c63e80e1c60bcc982e30ceeae163a9800e7/wrapt-2.0.1-cp313-cp313-win32.whl", hash = "sha256:d67956c676be5a24102c7407a71f4126d30de2a569a1c7871c9f3cabc94225d7", size = 58252, upload-time = "2025-11-07T00:44:17.814Z" }, + { url = "https://files.pythonhosted.org/packages/85/64/d3954e836ea67c4d3ad5285e5c8fd9d362fd0a189a2db622df457b0f4f6a/wrapt-2.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:9ca66b38dd642bf90c59b6738af8070747b610115a39af2498535f62b5cdc1c3", size = 60500, upload-time = "2025-11-07T00:44:15.561Z" }, + { url = "https://files.pythonhosted.org/packages/89/4e/3c8b99ac93527cfab7f116089db120fef16aac96e5f6cdb724ddf286086d/wrapt-2.0.1-cp313-cp313-win_arm64.whl", hash = "sha256:5a4939eae35db6b6cec8e7aa0e833dcca0acad8231672c26c2a9ab7a0f8ac9c8", size = 58993, upload-time = "2025-11-07T00:44:16.65Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f4/eff2b7d711cae20d220780b9300faa05558660afb93f2ff5db61fe725b9a/wrapt-2.0.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a52f93d95c8d38fed0669da2ebdb0b0376e895d84596a976c15a9eb45e3eccb3", size = 82028, upload-time = "2025-11-07T00:44:18.944Z" }, + { url = "https://files.pythonhosted.org/packages/0c/67/cb945563f66fd0f61a999339460d950f4735c69f18f0a87ca586319b1778/wrapt-2.0.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4e54bbf554ee29fcceee24fa41c4d091398b911da6e7f5d7bffda963c9aed2e1", size = 62949, upload-time = "2025-11-07T00:44:20.074Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ca/f63e177f0bbe1e5cf5e8d9b74a286537cd709724384ff20860f8f6065904/wrapt-2.0.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:908f8c6c71557f4deaa280f55d0728c3bca0960e8c3dd5ceeeafb3c19942719d", size = 63681, upload-time = "2025-11-07T00:44:21.345Z" }, + { url = "https://files.pythonhosted.org/packages/39/a1/1b88fcd21fd835dca48b556daef750952e917a2794fa20c025489e2e1f0f/wrapt-2.0.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e2f84e9af2060e3904a32cea9bb6db23ce3f91cfd90c6b426757cf7cc01c45c7", size = 152696, upload-time = "2025-11-07T00:44:24.318Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/d9185500c1960d9f5f77b9c0b890b7fc62282b53af7ad1b6bd779157f714/wrapt-2.0.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3612dc06b436968dfb9142c62e5dfa9eb5924f91120b3c8ff501ad878f90eb3", size = 158859, upload-time = "2025-11-07T00:44:25.494Z" }, + { url = "https://files.pythonhosted.org/packages/91/60/5d796ed0f481ec003220c7878a1d6894652efe089853a208ea0838c13086/wrapt-2.0.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6d2d947d266d99a1477cd005b23cbd09465276e302515e122df56bb9511aca1b", size = 146068, upload-time = "2025-11-07T00:44:22.81Z" }, + { url = "https://files.pythonhosted.org/packages/04/f8/75282dd72f102ddbfba137e1e15ecba47b40acff32c08ae97edbf53f469e/wrapt-2.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:7d539241e87b650cbc4c3ac9f32c8d1ac8a54e510f6dca3f6ab60dcfd48c9b10", size = 155724, upload-time = "2025-11-07T00:44:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/5a/27/fe39c51d1b344caebb4a6a9372157bdb8d25b194b3561b52c8ffc40ac7d1/wrapt-2.0.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:4811e15d88ee62dbf5c77f2c3ff3932b1e3ac92323ba3912f51fc4016ce81ecf", size = 144413, upload-time = "2025-11-07T00:44:27.939Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/9f6b643fe39d4505c7bf926d7c2595b7cb4b607c8c6b500e56c6b36ac238/wrapt-2.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c1c91405fcf1d501fa5d55df21e58ea49e6b879ae829f1039faaf7e5e509b41e", size = 150325, upload-time = "2025-11-07T00:44:29.29Z" }, + { url = "https://files.pythonhosted.org/packages/bb/b6/20ffcf2558596a7f58a2e69c89597128781f0b88e124bf5a4cadc05b8139/wrapt-2.0.1-cp313-cp313t-win32.whl", hash = "sha256:e76e3f91f864e89db8b8d2a8311d57df93f01ad6bb1e9b9976d1f2e83e18315c", size = 59943, upload-time = "2025-11-07T00:44:33.211Z" }, + { url = "https://files.pythonhosted.org/packages/87/6a/0e56111cbb3320151eed5d3821ee1373be13e05b376ea0870711f18810c3/wrapt-2.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:83ce30937f0ba0d28818807b303a412440c4b63e39d3d8fc036a94764b728c92", size = 63240, upload-time = "2025-11-07T00:44:30.935Z" }, + { url = "https://files.pythonhosted.org/packages/1d/54/5ab4c53ea1f7f7e5c3e7c1095db92932cc32fd62359d285486d00c2884c3/wrapt-2.0.1-cp313-cp313t-win_arm64.whl", hash = "sha256:4b55cacc57e1dc2d0991dbe74c6419ffd415fb66474a02335cb10efd1aa3f84f", size = 60416, upload-time = "2025-11-07T00:44:32.002Z" }, + { url = "https://files.pythonhosted.org/packages/73/81/d08d83c102709258e7730d3cd25befd114c60e43ef3891d7e6877971c514/wrapt-2.0.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:5e53b428f65ece6d9dad23cb87e64506392b720a0b45076c05354d27a13351a1", size = 78290, upload-time = "2025-11-07T00:44:34.691Z" }, + { url = "https://files.pythonhosted.org/packages/f6/14/393afba2abb65677f313aa680ff0981e829626fed39b6a7e3ec807487790/wrapt-2.0.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ad3ee9d0f254851c71780966eb417ef8e72117155cff04821ab9b60549694a55", size = 61255, upload-time = "2025-11-07T00:44:35.762Z" }, + { url = "https://files.pythonhosted.org/packages/c4/10/a4a1f2fba205a9462e36e708ba37e5ac95f4987a0f1f8fd23f0bf1fc3b0f/wrapt-2.0.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7b822c61ed04ee6ad64bc90d13368ad6eb094db54883b5dde2182f67a7f22c0", size = 61797, upload-time = "2025-11-07T00:44:37.22Z" }, + { url = "https://files.pythonhosted.org/packages/12/db/99ba5c37cf1c4fad35349174f1e38bd8d992340afc1ff27f526729b98986/wrapt-2.0.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7164a55f5e83a9a0b031d3ffab4d4e36bbec42e7025db560f225489fa929e509", size = 120470, upload-time = "2025-11-07T00:44:39.425Z" }, + { url = "https://files.pythonhosted.org/packages/30/3f/a1c8d2411eb826d695fc3395a431757331582907a0ec59afce8fe8712473/wrapt-2.0.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e60690ba71a57424c8d9ff28f8d006b7ad7772c22a4af432188572cd7fa004a1", size = 122851, upload-time = "2025-11-07T00:44:40.582Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8d/72c74a63f201768d6a04a8845c7976f86be6f5ff4d74996c272cefc8dafc/wrapt-2.0.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3cd1a4bd9a7a619922a8557e1318232e7269b5fb69d4ba97b04d20450a6bf970", size = 117433, upload-time = "2025-11-07T00:44:38.313Z" }, + { url = "https://files.pythonhosted.org/packages/c7/5a/df37cf4042cb13b08256f8e27023e2f9b3d471d553376616591bb99bcb31/wrapt-2.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b4c2e3d777e38e913b8ce3a6257af72fb608f86a1df471cb1d4339755d0a807c", size = 121280, upload-time = "2025-11-07T00:44:41.69Z" }, + { url = "https://files.pythonhosted.org/packages/54/34/40d6bc89349f9931e1186ceb3e5fbd61d307fef814f09fbbac98ada6a0c8/wrapt-2.0.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3d366aa598d69416b5afedf1faa539fac40c1d80a42f6b236c88c73a3c8f2d41", size = 116343, upload-time = "2025-11-07T00:44:43.013Z" }, + { url = "https://files.pythonhosted.org/packages/70/66/81c3461adece09d20781dee17c2366fdf0cb8754738b521d221ca056d596/wrapt-2.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c235095d6d090aa903f1db61f892fffb779c1eaeb2a50e566b52001f7a0f66ed", size = 119650, upload-time = "2025-11-07T00:44:44.523Z" }, + { url = "https://files.pythonhosted.org/packages/46/3a/d0146db8be8761a9e388cc9cc1c312b36d583950ec91696f19bbbb44af5a/wrapt-2.0.1-cp314-cp314-win32.whl", hash = "sha256:bfb5539005259f8127ea9c885bdc231978c06b7a980e63a8a61c8c4c979719d0", size = 58701, upload-time = "2025-11-07T00:44:48.277Z" }, + { url = "https://files.pythonhosted.org/packages/1a/38/5359da9af7d64554be63e9046164bd4d8ff289a2dd365677d25ba3342c08/wrapt-2.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:4ae879acc449caa9ed43fc36ba08392b9412ee67941748d31d94e3cedb36628c", size = 60947, upload-time = "2025-11-07T00:44:46.086Z" }, + { url = "https://files.pythonhosted.org/packages/aa/3f/96db0619276a833842bf36343685fa04f987dd6e3037f314531a1e00492b/wrapt-2.0.1-cp314-cp314-win_arm64.whl", hash = "sha256:8639b843c9efd84675f1e100ed9e99538ebea7297b62c4b45a7042edb84db03e", size = 59359, upload-time = "2025-11-07T00:44:47.164Z" }, + { url = "https://files.pythonhosted.org/packages/71/49/5f5d1e867bf2064bf3933bc6cf36ade23505f3902390e175e392173d36a2/wrapt-2.0.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:9219a1d946a9b32bb23ccae66bdb61e35c62773ce7ca6509ceea70f344656b7b", size = 82031, upload-time = "2025-11-07T00:44:49.4Z" }, + { url = "https://files.pythonhosted.org/packages/2b/89/0009a218d88db66ceb83921e5685e820e2c61b59bbbb1324ba65342668bc/wrapt-2.0.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fa4184e74197af3adad3c889a1af95b53bb0466bced92ea99a0c014e48323eec", size = 62952, upload-time = "2025-11-07T00:44:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/ae/18/9b968e920dd05d6e44bcc918a046d02afea0fb31b2f1c80ee4020f377cbe/wrapt-2.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c5ef2f2b8a53b7caee2f797ef166a390fef73979b15778a4a153e4b5fedce8fa", size = 63688, upload-time = "2025-11-07T00:44:52.248Z" }, + { url = "https://files.pythonhosted.org/packages/a6/7d/78bdcb75826725885d9ea26c49a03071b10c4c92da93edda612910f150e4/wrapt-2.0.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e042d653a4745be832d5aa190ff80ee4f02c34b21f4b785745eceacd0907b815", size = 152706, upload-time = "2025-11-07T00:44:54.613Z" }, + { url = "https://files.pythonhosted.org/packages/dd/77/cac1d46f47d32084a703df0d2d29d47e7eb2a7d19fa5cbca0e529ef57659/wrapt-2.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2afa23318136709c4b23d87d543b425c399887b4057936cd20386d5b1422b6fa", size = 158866, upload-time = "2025-11-07T00:44:55.79Z" }, + { url = "https://files.pythonhosted.org/packages/8a/11/b521406daa2421508903bf8d5e8b929216ec2af04839db31c0a2c525eee0/wrapt-2.0.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c72328f668cf4c503ffcf9434c2b71fdd624345ced7941bc6693e61bbe36bef", size = 146148, upload-time = "2025-11-07T00:44:53.388Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c0/340b272bed297baa7c9ce0c98ef7017d9c035a17a6a71dce3184b8382da2/wrapt-2.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3793ac154afb0e5b45d1233cb94d354ef7a983708cc3bb12563853b1d8d53747", size = 155737, upload-time = "2025-11-07T00:44:56.971Z" }, + { url = "https://files.pythonhosted.org/packages/f3/93/bfcb1fb2bdf186e9c2883a4d1ab45ab099c79cbf8f4e70ea453811fa3ea7/wrapt-2.0.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fec0d993ecba3991645b4857837277469c8cc4c554a7e24d064d1ca291cfb81f", size = 144451, upload-time = "2025-11-07T00:44:58.515Z" }, + { url = "https://files.pythonhosted.org/packages/d2/6b/dca504fb18d971139d232652656180e3bd57120e1193d9a5899c3c0b7cdd/wrapt-2.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:949520bccc1fa227274da7d03bf238be15389cd94e32e4297b92337df9b7a349", size = 150353, upload-time = "2025-11-07T00:44:59.753Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f6/a1de4bd3653afdf91d250ca5c721ee51195df2b61a4603d4b373aa804d1d/wrapt-2.0.1-cp314-cp314t-win32.whl", hash = "sha256:be9e84e91d6497ba62594158d3d31ec0486c60055c49179edc51ee43d095f79c", size = 60609, upload-time = "2025-11-07T00:45:03.315Z" }, + { url = "https://files.pythonhosted.org/packages/01/3a/07cd60a9d26fe73efead61c7830af975dfdba8537632d410462672e4432b/wrapt-2.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61c4956171c7434634401db448371277d07032a81cc21c599c22953374781395", size = 64038, upload-time = "2025-11-07T00:45:00.948Z" }, + { url = "https://files.pythonhosted.org/packages/41/99/8a06b8e17dddbf321325ae4eb12465804120f699cd1b8a355718300c62da/wrapt-2.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:35cdbd478607036fee40273be8ed54a451f5f23121bd9d4be515158f9498f7ad", size = 60634, upload-time = "2025-11-07T00:45:02.087Z" }, + { url = "https://files.pythonhosted.org/packages/15/d1/b51471c11592ff9c012bd3e2f7334a6ff2f42a7aed2caffcf0bdddc9cb89/wrapt-2.0.1-py3-none-any.whl", hash = "sha256:4d2ce1bf1a48c5277d7969259232b57645aae5686dba1eaeade39442277afbca", size = 44046, upload-time = "2025-11-07T00:45:32.116Z" }, +] From 2a94a45b34b8277cd0e701e2eb020022a30cf1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Otgaar?= Date: Tue, 13 Jan 2026 14:03:37 +0100 Subject: [PATCH 63/90] chore: adjust 'phase_id' to 'id' for correct payload --- .../agents/user_interrupt/user_interrupt_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 05af28a..108e821 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -168,7 +168,7 @@ class UserInterruptAgent(BaseAgent): new_phase_id = msg.body self.logger.info(f"Phase transition detected: {new_phase_id}") - payload = {"type": "phase_update", "phase_id": new_phase_id} + payload = {"type": "phase_update", "id": new_phase_id} await self._send_experiment_update(payload) case "goal_start": From f7669c021b097f5aa3e786dd0c4410d7fba5ed51 Mon Sep 17 00:00:00 2001 From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com> Date: Tue, 13 Jan 2026 17:04:44 +0100 Subject: [PATCH 64/90] feat: support force completed goals in semantic belief agent ref: N25B-427 --- .../agents/bdi/agentspeak_generator.py | 3 +- .../agents/bdi/text_belief_extractor_agent.py | 33 +++++++++++++++---- .../user_interrupt/user_interrupt_agent.py | 2 +- src/control_backend/schemas/belief_list.py | 4 +-- src/control_backend/schemas/program.py | 27 +++++++++++---- 5 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 9ab409d..21dc479 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -18,6 +18,7 @@ from control_backend.agents.bdi.agentspeak_ast import ( TriggerType, ) from control_backend.schemas.program import ( + BaseGoal, BasicNorm, ConditionalNorm, GestureAction, @@ -436,7 +437,7 @@ class AgentSpeakGenerator: @slugify.register @staticmethod - def _(g: Goal) -> str: + def _(g: BaseGoal) -> str: return AgentSpeakGenerator._slugify_str(g.name) @slugify.register diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index ebd9a65..b5fd266 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -12,7 +12,7 @@ from control_backend.schemas.belief_list import BeliefList, GoalList from control_backend.schemas.belief_message import Belief as InternalBelief from control_backend.schemas.belief_message import BeliefMessage from control_backend.schemas.chat_history import ChatHistory, ChatMessage -from control_backend.schemas.program import Goal, SemanticBelief +from control_backend.schemas.program import BaseGoal, SemanticBelief type JSONLike = None | bool | int | float | str | list["JSONLike"] | dict[str, "JSONLike"] @@ -62,6 +62,7 @@ class TextBeliefExtractorAgent(BaseAgent): self.goal_inferrer = GoalAchievementInferrer(self._llm) self._current_beliefs = BeliefState() self._current_goal_completions: dict[str, bool] = {} + self._force_completed_goals: set[BaseGoal] = set() self.conversation = ChatHistory(messages=[]) async def setup(self): @@ -118,13 +119,15 @@ class TextBeliefExtractorAgent(BaseAgent): case "goals": self._handle_goals_message(msg) await self._infer_goal_completions() + case "achieved_goals": + self._handle_goal_achieved_message(msg) case "conversation_history": if msg.body == "reset": - self._reset() + self._reset_phase() case _: self.logger.warning("Received unexpected message from %s", msg.sender) - def _reset(self): + def _reset_phase(self): self.conversation = ChatHistory(messages=[]) self.belief_inferrer.available_beliefs.clear() self._current_beliefs = BeliefState() @@ -158,7 +161,8 @@ class TextBeliefExtractorAgent(BaseAgent): return # Use only goals that can fail, as the others are always assumed to be completed - available_goals = [g for g in goals_list.goals if g.can_fail] + available_goals = {g for g in goals_list.goals if g.can_fail} + available_goals -= self._force_completed_goals self.goal_inferrer.goals = available_goals self.logger.debug( "Received %d failable goals from the program manager: %s", @@ -166,6 +170,23 @@ class TextBeliefExtractorAgent(BaseAgent): ", ".join(g.name for g in available_goals), ) + def _handle_goal_achieved_message(self, msg: InternalMessage): + # NOTE: When goals can be marked unachieved, remember to re-add them to the goal_inferrer + try: + goals_list = GoalList.model_validate_json(msg.body) + except ValidationError: + self.logger.warning( + "Received goal achieved message from the program manager, " + "but it is not a valid list of goals." + ) + return + + for goal in goals_list.goals: + self._force_completed_goals.add(goal) + self._current_goal_completions[f"achieved_{AgentSpeakGenerator.slugify(goal)}"] = True + + self.goal_inferrer.goals -= self._force_completed_goals + async def _user_said(self, text: str): """ Create a belief for the user's full speech. @@ -445,7 +466,7 @@ Respond with a JSON similar to the following, but with the property names as giv class GoalAchievementInferrer(SemanticBeliefInferrer): def __init__(self, llm: TextBeliefExtractorAgent.LLM): super().__init__(llm) - self.goals = [] + self.goals: set[BaseGoal] = set() async def infer_from_conversation(self, conversation: ChatHistory) -> dict[str, bool]: """ @@ -465,7 +486,7 @@ class GoalAchievementInferrer(SemanticBeliefInferrer): for goal, achieved in zip(self.goals, goals_achieved, strict=True) } - async def _infer_goal(self, conversation: ChatHistory, goal: Goal) -> bool: + async def _infer_goal(self, conversation: ChatHistory, goal: BaseGoal) -> bool: prompt = f"""{self._format_conversation(conversation)} Given the above conversation, what has the following goal been achieved? diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index d994121..4f12b34 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -310,7 +310,7 @@ class UserInterruptAgent(BaseAgent): async def _send_to_bdi_belief(self, asl_goal: str): """Send belief to BDI Core""" belief_name = f"achieved_{asl_goal}" - belief = Belief(name=belief_name) + belief = Belief(name=belief_name, arguments=None) self.logger.debug(f"Sending belief to BDI Core: {belief_name}") belief_message = BeliefMessage(create=[belief]) msg = InternalMessage( diff --git a/src/control_backend/schemas/belief_list.py b/src/control_backend/schemas/belief_list.py index b79247d..f3d6818 100644 --- a/src/control_backend/schemas/belief_list.py +++ b/src/control_backend/schemas/belief_list.py @@ -1,7 +1,7 @@ from pydantic import BaseModel +from control_backend.schemas.program import BaseGoal from control_backend.schemas.program import Belief as ProgramBelief -from control_backend.schemas.program import Goal class BeliefList(BaseModel): @@ -16,4 +16,4 @@ class BeliefList(BaseModel): class GoalList(BaseModel): - goals: list[Goal] + goals: list[BaseGoal] diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 3c8c7b4..d04abbb 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -15,6 +15,9 @@ class ProgramElement(BaseModel): name: str id: UUID4 + # To make program elements hashable + model_config = {"frozen": True} + class LogicalOperator(Enum): AND = "AND" @@ -105,23 +108,33 @@ class Plan(ProgramElement): steps: list[PlanElement] -class Goal(ProgramElement): +class BaseGoal(ProgramElement): """ - Represents an objective to be achieved. To reach the goal, we should execute - the corresponding plan. If we can fail to achieve a goal after executing the plan, - for example when the achieving of the goal is dependent on the user's reply, this means - that the achieved status will be set from somewhere else in the program. + Represents an objective to be achieved. This base version does not include a plan to achieve + this goal, and is used in semantic belief extraction. :ivar description: A description of the goal, used to determine if it has been achieved. - :ivar plan: The plan to execute. :ivar can_fail: Whether we can fail to achieve the goal after executing the plan. """ description: str = "" - plan: Plan can_fail: bool = True +class Goal(BaseGoal): + """ + Represents an objective to be achieved. To reach the goal, we should execute the corresponding + plan. It inherits from the BaseGoal a variable `can_fail`, which if true will cause the + completion to be determined based on the conversation. + + Instances of this goal are not hashable because a plan is not hashable. + + :ivar plan: The plan to execute. + """ + + plan: Plan + + type Action = SpeechAction | GestureAction | LLMAction From 43ac8ad69faaddda61b049e170e196ca4832d036 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 14 Jan 2026 10:58:41 +0100 Subject: [PATCH 65/90] chore: delete outdated files ref: N25B-446 --- src/control_backend/agents/bdi/asl_ast.py | 203 ----------- src/control_backend/agents/bdi/asl_gen.py | 425 ---------------------- 2 files changed, 628 deletions(-) delete mode 100644 src/control_backend/agents/bdi/asl_ast.py delete mode 100644 src/control_backend/agents/bdi/asl_gen.py diff --git a/src/control_backend/agents/bdi/asl_ast.py b/src/control_backend/agents/bdi/asl_ast.py deleted file mode 100644 index 104570b..0000000 --- a/src/control_backend/agents/bdi/asl_ast.py +++ /dev/null @@ -1,203 +0,0 @@ -import typing -from dataclasses import dataclass, field - -# --- Types --- - - -@dataclass -class BeliefLiteral: - """ - Represents a literal or atom. - Example: phase(1), user_said("hello"), ~started - """ - - functor: str - args: list[str] = field(default_factory=list) - negated: bool = False - - def __str__(self): - # In ASL, 'not' is usually for closed-world assumption (prolog style), - # '~' is for explicit negation in beliefs. - # For simplicity in behavior trees, we often use 'not' for conditions. - prefix = "not " if self.negated else "" - if not self.args: - return f"{prefix}{self.functor}" - - # Clean args to ensure strings are quoted if they look like strings, - # but usually the converter handles the quoting of string literals. - args_str = ", ".join(self.args) - return f"{prefix}{self.functor}({args_str})" - - -@dataclass -class GoalLiteral: - name: str - - def __str__(self): - return f"!{self.name}" - - -@dataclass -class ActionLiteral: - """ - Represents a step in a plan body. - Example: .say("Hello") or !achieve_goal - """ - - code: str - - def __str__(self): - return self.code - - -@dataclass -class BinaryOp: - """ - Represents logical operations. - Example: (A & B) | C - """ - - left: "Expression | str" - operator: typing.Literal["&", "|"] - right: "Expression | str" - - def __str__(self): - l_str = str(self.left) - r_str = str(self.right) - - if isinstance(self.left, BinaryOp): - l_str = f"({l_str})" - if isinstance(self.right, BinaryOp): - r_str = f"({r_str})" - - return f"{l_str} {self.operator} {r_str}" - - -Literal = BeliefLiteral | GoalLiteral | ActionLiteral -Expression = Literal | BinaryOp | str - - -@dataclass -class Rule: - """ - Represents an inference rule. - Example: head :- body. - """ - - head: Expression - body: Expression | None = None - - def __str__(self): - if not self.body: - return f"{self.head}." - return f"{self.head} :- {self.body}." - - -@dataclass -class PersistentRule: - """ - Represents an inference rule, where the inferred belief is persistent when formed. - """ - - head: Expression - body: Expression - - def __str__(self): - if not self.body: - raise Exception("Rule without body should not be persistent.") - - lines = [] - - if isinstance(self.body, BinaryOp): - lines.append(f"+{self.body.left}") - if self.body.operator == "&": - lines.append(f" : {self.body.right}") - lines.append(f" <- +{self.head}.") - if self.body.operator == "|": - lines.append(f"+{self.body.right}") - lines.append(f" <- +{self.head}.") - - return "\n".join(lines) - - -@dataclass -class Plan: - """ - Represents a plan. - Syntax: +trigger : context <- body. - """ - - trigger: BeliefLiteral | GoalLiteral - context: list[Expression] = field(default_factory=list) - body: list[ActionLiteral] = field(default_factory=list) - - def __str__(self): - # Indentation settings - INDENT = " " - ARROW = "\n <- " - COLON = "\n : " - - # Build Header - header = f"+{self.trigger}" - if self.context: - ctx_str = f" &\n{INDENT}".join(str(c) for c in self.context) - header += f"{COLON}{ctx_str}" - - # Case 1: Empty body - if not self.body: - return f"{header}." - - # Case 2: Short body (optional optimization, keeping it uniform usually better) - header += ARROW - - lines = [] - # We start the first action on the same line or next line. - # Let's put it on the next line for readability if there are multiple. - - if len(self.body) == 1: - return f"{header}{self.body[0]}." - - # First item - lines.append(f"{header}{self.body[0]};") - # Middle items - for item in self.body[1:-1]: - lines.append(f"{INDENT}{item};") - # Last item - lines.append(f"{INDENT}{self.body[-1]}.") - - return "\n".join(lines) - - -@dataclass -class AgentSpeakFile: - """ - Root element representing the entire generated file. - """ - - initial_beliefs: list[Rule] = field(default_factory=list) - inference_rules: list[Rule | PersistentRule] = field(default_factory=list) - plans: list[Plan] = field(default_factory=list) - - def __str__(self): - sections = [] - - if self.initial_beliefs: - sections.append("// --- Initial Beliefs & Facts ---") - sections.extend(str(rule) for rule in self.initial_beliefs) - sections.append("") - - if self.inference_rules: - sections.append("// --- Inference Rules ---") - sections.extend(str(rule) for rule in self.inference_rules if isinstance(rule, Rule)) - sections.append("") - sections.extend( - str(rule) for rule in self.inference_rules if isinstance(rule, PersistentRule) - ) - sections.append("") - - if self.plans: - sections.append("// --- Plans ---") - # Separate plans by a newline for readability - sections.extend(str(plan) + "\n" for plan in self.plans) - - return "\n".join(sections) diff --git a/src/control_backend/agents/bdi/asl_gen.py b/src/control_backend/agents/bdi/asl_gen.py deleted file mode 100644 index 8233a36..0000000 --- a/src/control_backend/agents/bdi/asl_gen.py +++ /dev/null @@ -1,425 +0,0 @@ -import asyncio -import time -from functools import singledispatchmethod - -from slugify import slugify - -from control_backend.agents.bdi import BDICoreAgent -from control_backend.agents.bdi.asl_ast import ( - ActionLiteral, - AgentSpeakFile, - BeliefLiteral, - BinaryOp, - Expression, - GoalLiteral, - PersistentRule, - Plan, - Rule, -) -from control_backend.agents.bdi.bdi_program_manager import test_program -from control_backend.schemas.program import ( - BasicBelief, - Belief, - ConditionalNorm, - GestureAction, - Goal, - InferredBelief, - KeywordBelief, - LLMAction, - LogicalOperator, - Phase, - Program, - ProgramElement, - SemanticBelief, - SpeechAction, -) - - -async def do_things(): - res = input("Wanna generate") - if res == "y": - program = AgentSpeakGenerator().generate(test_program) - filename = f"{int(time.time())}.asl" - with open(filename, "w") as f: - f.write(program) - else: - # filename = "0test.asl" - filename = "1766062491.asl" - bdi_agent = BDICoreAgent("BDICoreAgent", filename) - flag = asyncio.Event() - await bdi_agent.start() - await flag.wait() - - -def do_other_things(): - print(AgentSpeakGenerator().generate(test_program)) - - -class AgentSpeakGenerator: - """ - Converts a Pydantic Program behavior model into an AgentSpeak(L) AST, - then renders it to a string. - """ - - def generate(self, program: Program) -> str: - asl = AgentSpeakFile() - - self._generate_startup(program, asl) - - for i, phase in enumerate(program.phases): - next_phase = program.phases[i + 1] if i < len(program.phases) - 1 else None - - self._generate_phase_flow(phase, next_phase, asl) - - self._generate_norms(phase, asl) - - self._generate_goals(phase, asl) - - self._generate_triggers(phase, asl) - - self._generate_fallbacks(program, asl) - - return str(asl) - - # --- Section: Startup & Phase Management --- - - def _generate_startup(self, program: Program, asl: AgentSpeakFile): - if not program.phases: - return - - # Initial belief: phase(start). - asl.initial_beliefs.append(Rule(head=BeliefLiteral("phase", ['"start"']))) - - # Startup plan: +started : phase(start) <- -phase(start); +phase(first_id). - asl.plans.append( - Plan( - trigger=BeliefLiteral("started"), - context=[BeliefLiteral("phase", ['"start"'])], - body=[ - ActionLiteral('-phase("start")'), - ActionLiteral(f'+phase("{program.phases[0].id}")'), - ], - ) - ) - - # Initial plans: - asl.plans.append( - Plan( - trigger=GoalLiteral("generate_response_with_goal(Goal)"), - context=[BeliefLiteral("user_said", ["Message"])], - body=[ - ActionLiteral("+responded_this_turn"), - ActionLiteral(".findall(Norm, norm(Norm), Norms)"), - ActionLiteral(".reply_with_goal(Message, Norms, Goal)"), - ], - ) - ) - - def _generate_phase_flow(self, phase: Phase, next_phase: Phase | None, asl: AgentSpeakFile): - """Generates the main loop listener and the transition logic for this phase.""" - - # +user_said(Message) : phase(ID) <- !goal1; !goal2; !transition_phase. - goal_actions = [ActionLiteral("-responded_this_turn")] - goal_actions += [ - ActionLiteral(f"!check_{self._slugify_str(keyword)}") - for keyword in self._get_keyword_conditionals(phase) - ] - goal_actions += [ActionLiteral(f"!{self._slugify(g)}") for g in phase.goals] - goal_actions.append(ActionLiteral("!transition_phase")) - - asl.plans.append( - Plan( - trigger=BeliefLiteral("user_said", ["Message"]), - context=[BeliefLiteral("phase", [f'"{phase.id}"'])], - body=goal_actions, - ) - ) - - # +!transition_phase : phase(ID) <- -phase(ID); +(NEXT_ID). - next_id = str(next_phase.id) if next_phase else "end" - - transition_context = [BeliefLiteral("phase", [f'"{phase.id}"'])] - if phase.goals: - transition_context.append(BeliefLiteral(f"achieved_{self._slugify(phase.goals[-1])}")) - - asl.plans.append( - Plan( - trigger=GoalLiteral("transition_phase"), - context=transition_context, - body=[ - ActionLiteral(f'-phase("{phase.id}")'), - ActionLiteral(f'+phase("{next_id}")'), - ActionLiteral("user_said(Anything)"), - ActionLiteral("-+user_said(Anything)"), - ], - ) - ) - - def _get_keyword_conditionals(self, phase: Phase) -> list[str]: - res = [] - for belief in self._extract_basic_beliefs_from_phase(phase): - if isinstance(belief, KeywordBelief): - res.append(belief.keyword) - - return res - - # --- Section: Norms & Beliefs --- - - def _generate_norms(self, phase: Phase, asl: AgentSpeakFile): - for norm in phase.norms: - norm_slug = f'"{norm.norm}"' - head = BeliefLiteral("norm", [norm_slug]) - - # Base context is the phase - phase_lit = BeliefLiteral("phase", [f'"{phase.id}"']) - - if isinstance(norm, ConditionalNorm): - self._ensure_belief_inference(norm.condition, asl) - - condition_expr = self._belief_to_expr(norm.condition) - body = BinaryOp(phase_lit, "&", condition_expr) - else: - body = phase_lit - - asl.inference_rules.append(Rule(head=head, body=body)) - - def _ensure_belief_inference(self, belief: Belief, asl: AgentSpeakFile): - """ - Recursively adds rules to infer beliefs. - Checks strictly to avoid duplicates if necessary, - though ASL engines often handle redefinition or we can use a set to track processed IDs. - """ - if isinstance(belief, KeywordBelief): - pass - # # Rule: keyword_said("word") :- user_said(M) & .substring("word", M, P) & P >= 0. - # kwd_slug = f'"{belief.keyword}"' - # head = BeliefLiteral("keyword_said", [kwd_slug]) - # - # # Avoid duplicates - # if any(str(r.head) == str(head) for r in asl.inference_rules): - # return - # - # body = BinaryOp( - # BeliefLiteral("user_said", ["Message"]), - # "&", - # BinaryOp(f".substring({kwd_slug}, Message, Pos)", "&", "Pos >= 0"), - # ) - # - # asl.inference_rules.append(Rule(head=head, body=body)) - - elif isinstance(belief, InferredBelief): - self._ensure_belief_inference(belief.left, asl) - self._ensure_belief_inference(belief.right, asl) - - slug = self._slugify(belief) - head = BeliefLiteral(slug) - - if any(str(r.head) == str(head) for r in asl.inference_rules): - return - - op_char = "&" if belief.operator == LogicalOperator.AND else "|" - body = BinaryOp( - self._belief_to_expr(belief.left), op_char, self._belief_to_expr(belief.right) - ) - asl.inference_rules.append(PersistentRule(head=head, body=body)) - - def _belief_to_expr(self, belief: Belief) -> Expression: - if isinstance(belief, KeywordBelief): - return BeliefLiteral("keyword_said", [f'"{belief.keyword}"']) - else: - return BeliefLiteral(self._slugify(belief)) - - # --- Section: Goals --- - - def _generate_goals(self, phase: Phase, asl: AgentSpeakFile): - previous_goal: Goal | None = None - for goal in phase.goals: - self._generate_goal_plan_recursive(goal, str(phase.id), previous_goal, asl) - previous_goal = goal - - def _generate_goal_plan_recursive( - self, - goal: Goal, - phase_id: str, - previous_goal: Goal | None, - asl: AgentSpeakFile, - responded_needed: bool = True, - can_fail: bool = True, - ): - goal_slug = self._slugify(goal) - - # phase(ID) & not responded_this_turn & not achieved_goal - context = [ - BeliefLiteral("phase", [f'"{phase_id}"']), - ] - - if responded_needed: - context.append(BeliefLiteral("responded_this_turn", negated=True)) - if can_fail: - context.append(BeliefLiteral(f"achieved_{goal_slug}", negated=True)) - - if previous_goal: - prev_slug = self._slugify(previous_goal) - context.append(BeliefLiteral(f"achieved_{prev_slug}")) - - body_actions = [] - sub_goals_to_process = [] - - for step in goal.plan.steps: - if isinstance(step, Goal): - sub_slug = self._slugify(step) - body_actions.append(ActionLiteral(f"!{sub_slug}")) - sub_goals_to_process.append(step) - elif isinstance(step, SpeechAction): - body_actions.append(ActionLiteral(f'.say("{step.text}")')) - elif isinstance(step, GestureAction): - body_actions.append(ActionLiteral(f'.gesture("{step.gesture}")')) - elif isinstance(step, LLMAction): - body_actions.append(ActionLiteral(f'!generate_response_with_goal("{step.goal}")')) - - # Mark achievement - if not goal.can_fail: - body_actions.append(ActionLiteral(f"+achieved_{goal_slug}")) - - asl.plans.append(Plan(trigger=GoalLiteral(goal_slug), context=context, body=body_actions)) - asl.plans.append( - Plan(trigger=GoalLiteral(goal_slug), context=[], body=[ActionLiteral("true")]) - ) - - prev_sub = None - for sub_goal in sub_goals_to_process: - self._generate_goal_plan_recursive(sub_goal, phase_id, prev_sub, asl) - prev_sub = sub_goal - - # --- Section: Triggers --- - - def _generate_triggers(self, phase: Phase, asl: AgentSpeakFile): - for keyword in self._get_keyword_conditionals(phase): - asl.plans.append( - Plan( - trigger=GoalLiteral(f"check_{self._slugify_str(keyword)}"), - context=[ - ActionLiteral( - f'user_said(Message) & .substring("{keyword}", Message, Pos) & Pos >= 0' - ) - ], - body=[ - ActionLiteral(f'+keyword_said("{keyword}")'), - ActionLiteral(f'-keyword_said("{keyword}")'), - ], - ) - ) - asl.plans.append( - Plan( - trigger=GoalLiteral(f"check_{self._slugify_str(keyword)}"), - body=[ActionLiteral("true")], - ) - ) - - for trigger in phase.triggers: - self._ensure_belief_inference(trigger.condition, asl) - - trigger_belief_slug = self._belief_to_expr(trigger.condition) - - body_actions = [] - sub_goals = [] - - for step in trigger.plan.steps: - if isinstance(step, Goal): - sub_slug = self._slugify(step) - body_actions.append(ActionLiteral(f"!{sub_slug}")) - sub_goals.append(step) - elif isinstance(step, SpeechAction): - body_actions.append(ActionLiteral(f'.say("{step.text}")')) - elif isinstance(step, GestureAction): - body_actions.append( - ActionLiteral(f'.gesture("{step.gesture.type}", "{step.gesture.name}")') - ) - elif isinstance(step, LLMAction): - body_actions.append( - ActionLiteral(f'!generate_response_with_goal("{step.goal}")') - ) - - asl.plans.append( - Plan( - trigger=BeliefLiteral(trigger_belief_slug), - context=[BeliefLiteral("phase", [f'"{phase.id}"'])], - body=body_actions, - ) - ) - - # Recurse for triggered goals - prev_sub = None - for sub_goal in sub_goals: - self._generate_goal_plan_recursive( - sub_goal, str(phase.id), prev_sub, asl, False, False - ) - prev_sub = sub_goal - - # --- Section: Fallbacks --- - - def _generate_fallbacks(self, program: Program, asl: AgentSpeakFile): - asl.plans.append( - Plan(trigger=GoalLiteral("transition_phase"), context=[], body=[ActionLiteral("true")]) - ) - - # --- Helpers --- - - @singledispatchmethod - def _slugify(self, element: ProgramElement) -> str: - if element.name: - raise NotImplementedError("Cannot slugify this element.") - return self._slugify_str(element.name) - - @_slugify.register - def _(self, goal: Goal) -> str: - if goal.name: - return self._slugify_str(goal.name) - return f"goal_{goal.id.hex}" - - @_slugify.register - def _(self, kwb: KeywordBelief) -> str: - return f"keyword_said({kwb.keyword})" - - @_slugify.register - def _(self, sb: SemanticBelief) -> str: - return self._slugify_str(sb.description) - - @_slugify.register - def _(self, ib: InferredBelief) -> str: - return self._slugify_str(ib.name) - - def _slugify_str(self, text: str) -> str: - return slugify(text, separator="_", stopwords=["a", "an", "the", "we", "you", "I"]) - - def _extract_basic_beliefs_from_program(self, program: Program) -> list[BasicBelief]: - beliefs = [] - - for phase in program.phases: - beliefs.extend(self._extract_basic_beliefs_from_phase(phase)) - - return beliefs - - def _extract_basic_beliefs_from_phase(self, phase: Phase) -> list[BasicBelief]: - beliefs = [] - - for norm in phase.norms: - if isinstance(norm, ConditionalNorm): - beliefs += self._extract_basic_beliefs_from_belief(norm.condition) - - for trigger in phase.triggers: - beliefs += self._extract_basic_beliefs_from_belief(trigger.condition) - - return beliefs - - def _extract_basic_beliefs_from_belief(self, belief: Belief) -> list[BasicBelief]: - if isinstance(belief, InferredBelief): - return self._extract_basic_beliefs_from_belief( - belief.left - ) + self._extract_basic_beliefs_from_belief(belief.right) - return [belief] - - -if __name__ == "__main__": - asyncio.run(do_things()) - # do_other_things()y From ff24ab7a27a4de7581a57d48c259dce69ff706d0 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 14 Jan 2026 11:24:19 +0100 Subject: [PATCH 66/90] fix: default behavior and end phase ref: N25B-448 --- .../agents/bdi/agentspeak_generator.py | 25 ++++++++++++++-- .../agents/bdi/bdi_core_agent.py | 12 +------- .../agents/bdi/bdi_program_manager.py | 4 ++- .../agents/bdi/default_behavior.asl | 29 ++++++++++++++++--- 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 21dc479..68d1393 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -7,6 +7,7 @@ from control_backend.agents.bdi.agentspeak_ast import ( AstBinaryOp, AstExpression, AstLiteral, + AstNumber, AstPlan, AstProgram, AstRule, @@ -44,7 +45,11 @@ class AgentSpeakGenerator: def generate(self, program: Program) -> str: self._asp = AstProgram() - self._asp.rules.append(AstRule(self._astify(program.phases[0]))) + if program.phases: + self._asp.rules.append(AstRule(self._astify(program.phases[0]))) + else: + self._asp.rules.append(AstRule(AstLiteral("phase", [AstString("end")]))) + self._add_keyword_inference() self._add_default_plans() @@ -72,6 +77,7 @@ class AgentSpeakGenerator: self._add_reply_with_goal_plan() self._add_say_plan() self._add_reply_plan() + self._add_notify_cycle_plan() def _add_reply_with_goal_plan(self): self._asp.plans.append( @@ -134,6 +140,19 @@ class AgentSpeakGenerator: ) ) + def _add_notify_cycle_plan(self): + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("notify_cycle"), + [], + [ + AstStatement(StatementType.DO_ACTION, AstLiteral("notify_ui")), + AstStatement(StatementType.DO_ACTION, AstLiteral("wait", [AstNumber(1)])), + ], + ) + ) + def _process_phases(self, phases: list[Phase]) -> None: for curr_phase, next_phase in zip([None] + phases, phases + [None], strict=True): if curr_phase: @@ -148,7 +167,9 @@ class AgentSpeakGenerator: trigger_literal=AstLiteral("user_said", [AstVar("Message")]), context=[AstLiteral("phase", [AstString("end")])], body=[ - AstStatement(StatementType.DO_ACTION, AstLiteral("notify_user_said")), + AstStatement( + StatementType.DO_ACTION, AstLiteral("notify_user_said", [AstVar("Message")]) + ), AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("reply")), ], ) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 5b24c5d..9f8e2e4 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -342,14 +342,11 @@ class BDICoreAgent(BaseAgent): norm_update_message = InternalMessage( to=settings.agent_settings.user_interrupt_name, thread="active_norms_update", - body=norms, + body=str(norms), ) self.add_behavior(self.send(norm_update_message)) - self.logger.debug("Norms: %s", norms) - self.logger.debug("User text: %s", message_text) - self.add_behavior(self._send_to_llm(str(message_text), str(norms), "")) yield @@ -369,13 +366,6 @@ class BDICoreAgent(BaseAgent): ) self.add_behavior(self.send(norm_update_message)) - - self.logger.debug( - '"reply_with_goal" action called with message=%s, norms=%s, goal=%s', - message_text, - norms, - goal, - ) self.add_behavior(self._send_to_llm(str(message_text), str(norms), str(goal))) yield diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 25b7364..75ea757 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -279,8 +279,10 @@ class BDIProgramManager(BaseAgent): Initialize the agent. Connects the internal ZMQ SUB socket and subscribes to the 'program' topic. - Starts the background behavior to receive programs. + Starts the background behavior to receive programs. Initializes a default program. """ + await self._create_agentspeak_and_send_to_bdi(Program(phases=[])) + context = Context.instance() self.sub_socket = context.socket(zmq.SUB) diff --git a/src/control_backend/agents/bdi/default_behavior.asl b/src/control_backend/agents/bdi/default_behavior.asl index f7d1f95..b4d6682 100644 --- a/src/control_backend/agents/bdi/default_behavior.asl +++ b/src/control_backend/agents/bdi/default_behavior.asl @@ -1,13 +1,34 @@ -norm("Be friendly"). +phase("end"). +keyword_said(Keyword) :- (user_said(Message) & .substring(Keyword, Message, Pos)) & (Pos >= 0). + + ++!reply_with_goal(Goal) + : user_said(Message) + <- +responded_this_turn; + .findall(Norm, norm(Norm), Norms); + .reply_with_goal(Message, Norms, Goal). + ++!say(Text) + <- +responded_this_turn; + .say(Text). +!reply : user_said(Message) - <- .findall(Norm, norm(Norm), Norms); + <- +responded_this_turn; + .findall(Norm, norm(Norm), Norms); .reply(Message, Norms). ++!notify_cycle + <- .notify_ui; + .wait(1). + +user_said(Message) + : phase("end") <- .notify_user_said(Message); !reply. -+!transition_phase <- true. -+!check_triggers <- true. \ No newline at end of file ++!check_triggers + <- true. + ++!transition_phase + <- true. From 0794c549a8ccf570697eab10e1e3fe3c670df9ac Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 14 Jan 2026 11:27:29 +0100 Subject: [PATCH 67/90] chore: remove agentspeak file from tracking --- .gitignore | 2 + src/control_backend/agents/bdi/agentspeak.asl | 56 ------------------- 2 files changed, 2 insertions(+), 56 deletions(-) delete mode 100644 src/control_backend/agents/bdi/agentspeak.asl diff --git a/.gitignore b/.gitignore index f58719a..b6490a9 100644 --- a/.gitignore +++ b/.gitignore @@ -222,6 +222,8 @@ __marimo__/ docs/* !docs/conf.py +# Generated files +agentspeak.asl diff --git a/src/control_backend/agents/bdi/agentspeak.asl b/src/control_backend/agents/bdi/agentspeak.asl deleted file mode 100644 index 399566c..0000000 --- a/src/control_backend/agents/bdi/agentspeak.asl +++ /dev/null @@ -1,56 +0,0 @@ -phase("db4c68c3-0316-4905-a8db-22dd5bec7abf"). -keyword_said(Keyword) :- (user_said(Message) & .substring(Keyword, Message, Pos)) & (Pos >= 0). -norm("do nothing and make a little dance, do a little laugh") :- phase("db4c68c3-0316-4905-a8db-22dd5bec7abf") & keyword_said("hi"). - - -+!reply_with_goal(Goal) - : user_said(Message) - <- +responded_this_turn; - .findall(Norm, norm(Norm), Norms); - .reply_with_goal(Message, Norms, Goal). - -+!say(Text) - <- +responded_this_turn; - .say(Text). - -+!reply - : user_said(Message) - <- +responded_this_turn; - .findall(Norm, norm(Norm), Norms); - .reply(Message, Norms). - -+user_said(Message) - : phase("db4c68c3-0316-4905-a8db-22dd5bec7abf") - <- .notify_user_said(Message); - -responded_this_turn; - !check_triggers; - !transition_phase. - -+!check_triggers - : phase("db4c68c3-0316-4905-a8db-22dd5bec7abf") & - semantic_hello - <- .notify_trigger_start("trigger_"); - .notify_trigger_end("trigger_"). - -+!trigger_ - <- .notify_trigger_start("trigger_"); - .notify_trigger_end("trigger_"). - -+!transition_phase - : phase("db4c68c3-0316-4905-a8db-22dd5bec7abf") & - not responded_this_turn - <- .notify_transition_phase("db4c68c3-0316-4905-a8db-22dd5bec7abf", "end"); - -phase("db4c68c3-0316-4905-a8db-22dd5bec7abf"); - +phase("end"); - ?user_said(Message); - -+user_said(Message). - -+user_said(Message) - : phase("end") - <- !reply. - -+!check_triggers - <- true. - -+!transition_phase - <- true. From 8f6662e64a72376298ff0c87fbf0d7b0df721423 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 14 Jan 2026 13:22:51 +0100 Subject: [PATCH 68/90] feat: phase transitions ref: N25B-446 --- .../agents/bdi/agentspeak_generator.py | 23 +++++++++++++-- .../agents/bdi/bdi_core_agent.py | 28 +++++++++---------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 68d1393..11bb2c8 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -197,10 +197,12 @@ class AgentSpeakGenerator: self._astify(to_phase) if to_phase else AstLiteral("phase", [AstString("end")]) ) - context = [from_phase_ast] + check_context = [from_phase_ast] if from_phase: for goal in from_phase.goals: - context.append(self._astify(goal, achieved=True)) + check_context.append(self._astify(goal, achieved=True)) + + force_context = [from_phase_ast] body = [ AstStatement( @@ -229,8 +231,23 @@ class AgentSpeakGenerator: # ] # ) + # Check self._asp.plans.append( - AstPlan(TriggerType.ADDED_GOAL, AstLiteral("transition_phase"), context, body) + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("transition_phase"), + check_context, + [ + AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("force_transition_phase")), + ], + ) + ) + + # Force + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, AstLiteral("force_transition_phase"), force_context, body + ) ) def _process_norm(self, norm: Norm, phase: Phase) -> None: diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 9f8e2e4..8eb4d23 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -156,8 +156,7 @@ class BDICoreAgent(BaseAgent): ) await self.send(out_msg) case settings.agent_settings.user_interrupt_name: - content = msg.body - self.logger.debug("Received user interruption: %s", content) + self.logger.debug("Received user interruption: %s", msg) match msg.thread: case "force_phase_transition": @@ -166,6 +165,8 @@ class BDICoreAgent(BaseAgent): self._force_trigger(msg.body) case "force_norm": self._force_norm(msg.body) + case "force_next_phase": + self._force_next_phase() case _: self.logger.warning("Received unknow user interruption: %s", msg) @@ -304,26 +305,21 @@ class BDICoreAgent(BaseAgent): self.logger.debug(f"Set goal !{self.format_belief_string(name, args)}.") def _force_trigger(self, name: str): - self.bdi_agent.call( - agentspeak.Trigger.addition, - agentspeak.GoalType.achievement, - agentspeak.Literal(name), - agentspeak.runtime.Intention(), - ) + self._set_goal(name) self.logger.info("Manually forced trigger %s.", name) # TODO: make this compatible for critical norms def _force_norm(self, name: str): - self.bdi_agent.call( - agentspeak.Trigger.addition, - agentspeak.GoalType.belief, - agentspeak.Literal(f"force_{name}"), - agentspeak.runtime.Intention(), - ) + self._add_belief(f"force_{name}") self.logger.info("Manually forced norm %s.", name) + def _force_next_phase(self): + self._set_goal("force_transition_phase") + + self.logger.info("Manually forced phase transition.") + def _add_custom_actions(self) -> None: """ Add any custom actions here. Inside `@self.actions.add()`, the first argument is @@ -520,6 +516,10 @@ class BDICoreAgent(BaseAgent): yield + @self.actions.add(".notify_ui", 0) + def _notify_ui(agent, term, intention): + pass + async def _send_to_llm(self, text: str, norms: str, goals: str): """ Sends a text query to the LLM agent asynchronously. From 39e1bb1ead28c7acfa870b53fa3bbf1725400f2a Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Wed, 14 Jan 2026 15:28:29 +0100 Subject: [PATCH 69/90] fix: sync issues ref: N25B-447 --- .../agents/bdi/agentspeak_generator.py | 20 +++++++++++++++++-- .../agents/bdi/bdi_core_agent.py | 20 ++++++++----------- .../agents/bdi/bdi_program_manager.py | 9 +++++++++ .../user_interrupt/user_interrupt_agent.py | 9 +++++---- src/control_backend/core/agent_system.py | 11 ++++++---- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 11bb2c8..ed6f787 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -50,6 +50,8 @@ class AgentSpeakGenerator: else: self._asp.rules.append(AstRule(AstLiteral("phase", [AstString("end")]))) + self._asp.rules.append(AstRule(AstLiteral("!notify_cycle"))) + self._add_keyword_inference() self._add_default_plans() @@ -147,8 +149,18 @@ class AgentSpeakGenerator: AstLiteral("notify_cycle"), [], [ - AstStatement(StatementType.DO_ACTION, AstLiteral("notify_ui")), - AstStatement(StatementType.DO_ACTION, AstLiteral("wait", [AstNumber(1)])), + AstStatement( + StatementType.DO_ACTION, + AstLiteral( + "findall", + [AstVar("Norm"), AstLiteral("norm", [AstVar("Norm")]), AstVar("Norms")], + ), + ), + AstStatement( + StatementType.DO_ACTION, AstLiteral("notify_norms", [AstVar("Norms")]) + ), + AstStatement(StatementType.DO_ACTION, AstLiteral("wait", [AstNumber(100)])), + AstStatement(StatementType.ACHIEVE_GOAL, AstLiteral("notify_cycle")), ], ) ) @@ -365,6 +377,10 @@ class AgentSpeakGenerator: if isinstance(step, Goal): step.can_fail = False # triggers are continuous sequence subgoals.append(step) + + # Arbitrary wait for UI to display nicely + body.append(AstStatement(StatementType.DO_ACTION, AstLiteral("wait", [AstNumber(2000)]))) + body.append( AstStatement( StatementType.DO_ACTION, diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 8eb4d23..0c217dc 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -107,7 +107,6 @@ class BDICoreAgent(BaseAgent): if not maybe_more_work: deadline = self.bdi_agent.shortest_deadline() if deadline: - self.logger.debug("Sleeping until %s", deadline) await asyncio.sleep(deadline - time.time()) maybe_more_work = True else: @@ -335,14 +334,6 @@ class BDICoreAgent(BaseAgent): message_text = agentspeak.grounded(term.args[0], intention.scope) norms = agentspeak.grounded(term.args[1], intention.scope) - norm_update_message = InternalMessage( - to=settings.agent_settings.user_interrupt_name, - thread="active_norms_update", - body=str(norms), - ) - - self.add_behavior(self.send(norm_update_message)) - self.add_behavior(self._send_to_llm(str(message_text), str(norms), "")) yield @@ -355,14 +346,20 @@ class BDICoreAgent(BaseAgent): message_text = agentspeak.grounded(term.args[0], intention.scope) norms = agentspeak.grounded(term.args[1], intention.scope) goal = agentspeak.grounded(term.args[2], intention.scope) + self.add_behavior(self._send_to_llm(str(message_text), str(norms), str(goal))) + yield + + @self.actions.add(".notify_norms", 1) + def _notify_norms(agent, term, intention): + norms = agentspeak.grounded(term.args[0], intention.scope) + norm_update_message = InternalMessage( to=settings.agent_settings.user_interrupt_name, thread="active_norms_update", body=str(norms), ) - self.add_behavior(self.send(norm_update_message)) - self.add_behavior(self._send_to_llm(str(message_text), str(norms), str(goal))) + self.add_behavior(self.send(norm_update_message, should_log=False)) yield @self.actions.add(".say", 1) @@ -473,7 +470,6 @@ class BDICoreAgent(BaseAgent): body=str(trigger_name), ) - # TODO: check with Pim self.add_behavior(self.send(msg)) yield diff --git a/src/control_backend/agents/bdi/bdi_program_manager.py b/src/control_backend/agents/bdi/bdi_program_manager.py index 75ea757..730c8e5 100644 --- a/src/control_backend/agents/bdi/bdi_program_manager.py +++ b/src/control_backend/agents/bdi/bdi_program_manager.py @@ -97,6 +97,15 @@ class BDIProgramManager(BaseAgent): if new == "end": self._phase = None + # Notify user interaction agent + msg = InternalMessage( + to=settings.agent_settings.user_interrupt_name, + thread="transition_phase", + body="end", + ) + self.logger.info("Transitioned to end phase, notifying UserInterruptAgent.") + + self.add_behavior(self.send(msg)) return for phase in self._program.phases: diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 4f12b34..deddbba 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -214,8 +214,8 @@ class UserInterruptAgent(BaseAgent): payload = {"type": "cond_norms_state_update", "norms": updates} - await self._send_experiment_update(payload) - self.logger.debug(f"Broadcasted state for {len(updates)} conditional norms.") + await self._send_experiment_update(payload, should_log=False) + # self.logger.debug(f"Broadcasted state for {len(updates)} conditional norms.") def _create_mapping(self, program_json: str): """ @@ -259,7 +259,7 @@ class UserInterruptAgent(BaseAgent): except Exception as e: self.logger.error(f"Mapping failed: {e}") - async def _send_experiment_update(self, data): + async def _send_experiment_update(self, data, should_log: bool = True): """ Sends an update to the 'experiment' topic. The SSE endpoint will pick this up and push it to the UI. @@ -268,7 +268,8 @@ class UserInterruptAgent(BaseAgent): topic = b"experiment" body = json.dumps(data).encode("utf-8") await self.pub_socket.send_multipart([topic, body]) - self.logger.debug(f"Sent experiment update: {data}") + if should_log: + self.logger.debug(f"Sent experiment update: {data}") async def _send_to_speech_agent(self, text_to_say: str): """ diff --git a/src/control_backend/core/agent_system.py b/src/control_backend/core/agent_system.py index 2d8492a..e3c8dc4 100644 --- a/src/control_backend/core/agent_system.py +++ b/src/control_backend/core/agent_system.py @@ -120,7 +120,7 @@ class BaseAgent(ABC): task.cancel() self.logger.info(f"Agent {self.name} stopped") - async def send(self, message: InternalMessage): + async def send(self, message: InternalMessage, should_log: bool = True): """ Send a message to another agent. @@ -142,13 +142,17 @@ class BaseAgent(ABC): if target: await target.inbox.put(message) - self.logger.debug(f"Sent message {message.body} to {message.to} via regular inbox.") + if should_log: + self.logger.debug( + f"Sent message {message.body} to {message.to} via regular inbox." + ) else: # Apparently target agent is on a different process, send via ZMQ topic = f"internal/{receiver}".encode() body = message.model_dump_json().encode() await self._internal_pub_socket.send_multipart([topic, body]) - self.logger.debug(f"Sent message {message.body} to {message.to} via ZMQ.") + if should_log: + self.logger.debug(f"Sent message {message.body} to {message.to} via ZMQ.") async def _process_inbox(self): """ @@ -158,7 +162,6 @@ class BaseAgent(ABC): """ while self._running: msg = await self.inbox.get() - self.logger.debug(f"Received message from {msg.sender}.") await self.handle_message(msg) async def _receive_internal_zmq_loop(self): From 041fc4ab6e01183512345d83887e9df5e4d58c17 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Thu, 15 Jan 2026 09:02:52 +0100 Subject: [PATCH 70/90] chore: cond_norms unachieve and via belief msg --- .../user_interrupt/user_interrupt_agent.py | 140 ++++++++++-------- 1 file changed, 80 insertions(+), 60 deletions(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index deddbba..0bde563 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -26,7 +26,7 @@ class UserInterruptAgent(BaseAgent): - Send a prioritized message to the `RobotSpeechAgent` - Send a prioritized gesture to the `RobotGestureAgent` - - Send a belief override to the `BDIProgramManager`in order to activate a + - Send a belief override to the `BDI Core` in order to activate a trigger/conditional norm or complete a goal. Prioritized actions clear the current RI queue before inserting the new item, @@ -75,7 +75,9 @@ class UserInterruptAgent(BaseAgent): These are the different types and contexts: - type: "speech", context: string that the robot has to say. - type: "gesture", context: single gesture name that the robot has to perform. - - type: "override", context: belief_id that overrides the goal/trigger/conditional norm. + - type: "override", context: id that belongs to the goal/trigger/conditional norm. + - type: "override_unachieve", context: id that belongs to the conditional norm to unachieve. + - type: "next_phase", context: None, indicates to the BDI Core to - type: "pause", context: boolean indicating whether to pause - type: "reset_phase", context: None, indicates to the BDI Core to - type: "reset_experiment", context: None, indicates to the BDI Core to @@ -93,68 +95,82 @@ class UserInterruptAgent(BaseAgent): self.logger.debug("Received event type %s", event_type) - if event_type == "speech": - await self._send_to_speech_agent(event_context) - self.logger.info( - "Forwarded button press (speech) with context '%s' to RobotSpeechAgent.", - event_context, - ) - elif event_type == "gesture": - await self._send_to_gesture_agent(event_context) - self.logger.info( - "Forwarded button press (gesture) with context '%s' to RobotGestureAgent.", - event_context, - ) - elif event_type == "override": - ui_id = str(event_context) - if asl_trigger := self._trigger_map.get(ui_id): - await self._send_to_bdi("force_trigger", asl_trigger) + match event_type: + case "speech": + await self._send_to_speech_agent(event_context) self.logger.info( - "Forwarded button press (override) with context '%s' to BDI Core.", + "Forwarded button press (speech) with context '%s' to RobotSpeechAgent.", event_context, ) - elif asl_cond_norm := self._cond_norm_map.get(ui_id): - await self._send_to_bdi("force_norm", asl_cond_norm) + case "gesture": + await self._send_to_gesture_agent(event_context) self.logger.info( - "Forwarded button press (override) with context '%s' to BDIProgramManager.", + "Forwarded button press (gesture) with context '%s' to RobotGestureAgent.", event_context, ) - elif asl_goal := self._goal_map.get(ui_id): - await self._send_to_bdi_belief(asl_goal) - self.logger.info( - "Forwarded button press (override) with context '%s' to BDI Core.", + case "override": + ui_id = str(event_context) + if asl_trigger := self._trigger_map.get(ui_id): + await self._send_to_bdi("force_trigger", asl_trigger) + self.logger.info( + "Forwarded button press (override) with context '%s' to BDI Core.", + event_context, + ) + elif asl_cond_norm := self._cond_norm_map.get(ui_id): + await self._send_to_bdi_belief(asl_cond_norm) + self.logger.info( + "Forwarded button press (override) with context '%s' to BDI Core.", + event_context, + ) + elif asl_goal := self._goal_map.get(ui_id): + await self._send_to_bdi_belief(asl_goal) + self.logger.info( + "Forwarded button press (override) with context '%s' to BDI Core.", + event_context, + ) + # Send achieve_goal to program manager to update semantic belief extractor + goal_achieve_msg = InternalMessage( + to=settings.agent_settings.bdi_program_manager_name, + thread="achieve_goal", + body=ui_id, + ) + + await self.send(goal_achieve_msg) + else: + self.logger.warning("Could not determine which element to override.") + case "override_unachieve": + ui_id = str(event_context) + if asl_cond_norm := self._cond_norm_map.get(ui_id): + await self._send_to_bdi_belief(asl_cond_norm, True) + self.logger.info( + "Forwarded button press (override_unachieve)" + "with context '%s' to BDI Core.", + event_context, + ) + else: + self.logger.warning( + "Could not determine which conditional norm to unachieve." + ) + + case "pause": + self.logger.debug( + "Received pause/resume button press with context '%s'.", event_context + ) + await self._send_pause_command(event_context) + if event_context: + self.logger.info("Sent pause command.") + else: + self.logger.info("Sent resume command.") + + case "next_phase" | "reset_phase" | "reset_experiment": + await self._send_experiment_control_to_bdi_core(event_type) + case _: + self.logger.warning( + "Received button press with unknown type '%s' (context: '%s').", + event_type, event_context, ) - goal_achieve_msg = InternalMessage( - to=settings.agent_settings.bdi_program_manager_name, - thread="achieve_goal", - body=ui_id, - ) - - await self.send(goal_achieve_msg) - else: - self.logger.warning("Could not determine which element to override.") - - elif event_type == "pause": - self.logger.debug( - "Received pause/resume button press with context '%s'.", event_context - ) - await self._send_pause_command(event_context) - if event_context: - self.logger.info("Sent pause command.") - else: - self.logger.info("Sent resume command.") - - elif event_type in ["next_phase", "reset_phase", "reset_experiment"]: - await self._send_experiment_control_to_bdi_core(event_type) - else: - self.logger.warning( - "Received button press with unknown type '%s' (context: '%s').", - event_type, - event_context, - ) - async def handle_message(self, msg: InternalMessage): """ Handle commands received from other internal Python agents. @@ -195,9 +211,10 @@ class UserInterruptAgent(BaseAgent): await self._send_experiment_update(payload) self.logger.info(f"UI Update: Goal {goal_name} started (ID: {ui_id})") case "active_norms_update": - norm_list = [s.strip("() '\",") for s in msg.body.split(",") if s.strip("() '\",")] - - await self._broadcast_cond_norms(norm_list) + active_norms_asl = [ + s.strip("() '\",") for s in msg.body.split(",") if s.strip("() '\",") + ] + await self._broadcast_cond_norms(active_norms_asl) case _: self.logger.debug(f"Received internal message on unhandled thread: {msg.thread}") @@ -308,12 +325,15 @@ class UserInterruptAgent(BaseAgent): await self.send(msg) self.logger.info(f"Directly forced {thread} in BDI: {body}") - async def _send_to_bdi_belief(self, asl_goal: str): + async def _send_to_bdi_belief(self, asl_goal: str, unachieve: bool = False): """Send belief to BDI Core""" belief_name = f"achieved_{asl_goal}" belief = Belief(name=belief_name, arguments=None) self.logger.debug(f"Sending belief to BDI Core: {belief_name}") - belief_message = BeliefMessage(create=[belief]) + # Conditional norms are unachieved by removing the belief + belief_message = ( + BeliefMessage(delete=[belief]) if unachieve else BeliefMessage(create=[belief]) + ) msg = InternalMessage( to=settings.agent_settings.bdi_core_name, thread="beliefs", From 0771b0d607fb739aa89f11846ebc1d6502d43c8d Mon Sep 17 00:00:00 2001 From: Storm Date: Fri, 16 Jan 2026 09:50:59 +0100 Subject: [PATCH 71/90] feat: implemented visual emotion recogntion agent ref: N25B-393 --- .../visual_emotion_recognition_agent.py | 83 ++++++++++++++++--- .../visual_emotion_recognizer.py | 23 ++++- src/control_backend/main.py | 8 ++ 3 files changed, 99 insertions(+), 15 deletions(-) diff --git a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py index f301c6a..1087138 100644 --- a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py @@ -1,13 +1,17 @@ import asyncio import zmq import zmq.asyncio as azmq +import numpy as np +import cv2 +from collections import defaultdict, Counter +import time from control_backend.agents import BaseAgent from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognizer import DeepFaceEmotionRecognizer from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings -# START FROM RI? +# START FROM RI COMMUNICATION AGENT? class VisualEmotionRecognitionAgent(BaseAgent): def __init__(self, socket_address: str, socket_bind: bool = False, timeout_ms: int = 1000): @@ -32,19 +36,76 @@ class VisualEmotionRecognitionAgent(BaseAgent): self.video_in_socket.setsockopt(zmq.RCVTIMEO, self.timeout_ms) self.video_in_socket.setsockopt(zmq.CONFLATE, 1) - self.add_behavior(self.retrieve_frame()) + self.add_behavior(self.emotion_update_loop()) - async def retrieve_frame(self): + async def emotion_update_loop(self): """ Retrieve a video frame from the input socket. :return: The received video frame, or None if timeout occurs. """ - await asyncio.sleep(1) # Yield control to the event loop - try: - frame = await self.video_in_socket.recv() - # detected_emotions contains a list of dictionaries as follows: - detected_emotions = self.emotion_recognizer.detect(frame) - except zmq.Again: - self.logger.debug("No video frame received within timeout.") - return None \ No newline at end of file + window_duration = 1 # seconds + next_window_time = time.time() + window_duration + + # To detect false positives + # Minimal number of frames a face has to be detected to consider it valid + # Can also reduce false positives by ignoring faces that are too small; not implemented + # Also use face confidence thresholding in recognizer + min_frames_required = 2 + + face_stats = defaultdict(Counter) + + prev_dominant_emotions = set() + + while self._running: + try: + frame_bytes = await self.video_in_socket.recv() + + # Convert bytes to a numpy buffer + nparr = np.frombuffer(frame_bytes, np.uint8) + + # Decode image into the generic Numpy Array DeepFace expects + frame_image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) + + if frame_image is None: + # Could not decode image, skip this frame + continue + + # Get the dominant emotion from each face + current_emotions = self.emotion_recognizer.sorted_dominant_emotions(frame_image) + # Update emotion counts for each detected face + for i, emotion in enumerate(current_emotions): + face_stats[i][emotion] += 1 + + # If window duration has passed, process the collected stats + if time.time() >= next_window_time: + + window_dominant_emotions = set() + # Determine dominant emotion for each face in the window + for _, counter in face_stats.items(): + total_detections = sum(counter.values()) + + if total_detections >= min_frames_required: + dominant_emotion = counter.most_common(1)[0][0] + window_dominant_emotions.add(dominant_emotion) + + await self.update_emotions(prev_dominant_emotions, window_dominant_emotions) + + prev_dominant_emotions = window_dominant_emotions + face_stats.clear() + next_window_time = time.time() + window_duration + + except zmq.Again: + self.logger.warning("No video frame received within timeout.") + + async def update_emotions(self, prev_emotions, emotions): + # Remove emotions that are no longer present + emotions_to_remove = prev_emotions - emotions + for emotion in emotions_to_remove: + self.logger.info(f"Emotion '{emotion}' has disappeared.") + + # Add new emotions that have appeared + new_emotions = emotions - prev_emotions + for emotion in new_emotions: + self.logger.info(f"New emotion detected: '{emotion}'") + \ No newline at end of file diff --git a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py index 069441e..06e7e4d 100644 --- a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py +++ b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py @@ -1,6 +1,7 @@ import abc from deepface import DeepFace import numpy as np +from collections import Counter class VisualEmotionRecognizer(abc.ABC): @abc.abstractmethod @@ -9,7 +10,7 @@ class VisualEmotionRecognizer(abc.ABC): pass @abc.abstractmethod - def detect(self, image): + def sorted_dominant_emotions(self, image): """Recognize emotion from the given image. :param image: The input image for emotion recognition. @@ -29,7 +30,21 @@ class DeepFaceEmotionRecognizer(VisualEmotionRecognizer): # the model DeepFace.analyze(dummy_img, actions=['emotion'], enforce_detection=False) print("Deepface Emotion Model loaded.") + + def sorted_dominant_emotions(self, image): + analysis = DeepFace.analyze(image, + actions=['emotion'], + enforce_detection=False + ) + + # Sort faces by x coordinate to maintain left-to-right order + analysis.sort(key=lambda face: face['region']['x']) - def detect(self, image): - analysis = DeepFace.analyze(image, actions=['emotion'], enforce_detection=False) - return analysis['dominant_emotion'] \ No newline at end of file + analysis = [face for face in analysis if face['face_confidence'] >= 0.90] + + # Return list of (dominant_emotion, face_confidence) tuples + dominant_emotions = [face['dominant_emotion'] for face in analysis] + return dominant_emotions + + + diff --git a/src/control_backend/main.py b/src/control_backend/main.py index 3509cbc..ce2b852 100644 --- a/src/control_backend/main.py +++ b/src/control_backend/main.py @@ -40,6 +40,7 @@ from control_backend.agents.communication import RICommunicationAgent from control_backend.agents.llm import LLMAgent # User Interrupt Agent +from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognition_agent import VisualEmotionRecognitionAgent from control_backend.agents.user_interrupt.user_interrupt_agent import UserInterruptAgent # Other backend imports @@ -147,6 +148,13 @@ async def lifespan(app: FastAPI): "name": settings.agent_settings.user_interrupt_name, }, ), + # TODO: Spawn agent from RI Communication Agent + "VisualEmotionRecognitionAgent": ( + VisualEmotionRecognitionAgent, + { + "socket_address": "tcp://localhost:5556", # TODO: move to settings + }, + ), } agents = [] From b1c18abffd2d15cfa3473a56ebb2198c690c79d7 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Fri, 16 Jan 2026 13:11:41 +0100 Subject: [PATCH 72/90] test: bunch of tests Written with AI, still need to check them ref: N25B-449 --- src/control_backend/agents/bdi/__init__.py | 3 - .../agents/bdi/agentspeak_ast.py | 10 +- .../agents/bdi/belief_collector_agent.py | 152 ----------- .../communication/ri_communication_agent.py | 4 +- src/control_backend/main.py | 2 + .../actuation/test_robot_gesture_agent.py | 71 ++++- .../actuation/test_robot_speech_agent.py | 12 +- test/unit/agents/bdi/test_agentspeak_ast.py | 186 +++++++++++++ .../agents/bdi/test_agentspeak_generator.py | 187 +++++++++++++ test/unit/agents/bdi/test_bdi_core_agent.py | 258 +++++++++++++++++- .../agents/bdi/test_bdi_program_manager.py | 213 +++++++++++++-- test/unit/agents/bdi/test_belief_collector.py | 135 --------- .../agents/bdi/test_text_belief_extractor.py | 156 ++++++++++- .../test_ri_communication_agent.py | 70 ++++- test/unit/agents/llm/test_llm_agent.py | 69 +++-- .../test_transcription_agent.py | 21 +- .../perception/vad_agent/test_vad_agent.py | 153 +++++++++++ .../vad_agent/test_vad_streaming.py | 49 ++++ test/unit/agents/test_base.py | 24 ++ .../user_interrupt/test_user_interrupt.py | 209 ++++++++++++-- .../api/v1/endpoints/test_user_interact.py | 96 +++++++ test/unit/test_main_sockets.py | 40 +++ 22 files changed, 1747 insertions(+), 373 deletions(-) delete mode 100644 src/control_backend/agents/bdi/belief_collector_agent.py create mode 100644 test/unit/agents/bdi/test_agentspeak_ast.py create mode 100644 test/unit/agents/bdi/test_agentspeak_generator.py delete mode 100644 test/unit/agents/bdi/test_belief_collector.py create mode 100644 test/unit/agents/perception/vad_agent/test_vad_agent.py create mode 100644 test/unit/agents/test_base.py create mode 100644 test/unit/api/v1/endpoints/test_user_interact.py create mode 100644 test/unit/test_main_sockets.py diff --git a/src/control_backend/agents/bdi/__init__.py b/src/control_backend/agents/bdi/__init__.py index 8d45440..d6f5124 100644 --- a/src/control_backend/agents/bdi/__init__.py +++ b/src/control_backend/agents/bdi/__init__.py @@ -1,8 +1,5 @@ from control_backend.agents.bdi.bdi_core_agent import BDICoreAgent as BDICoreAgent -from .belief_collector_agent import ( - BDIBeliefCollectorAgent as BDIBeliefCollectorAgent, -) from .text_belief_extractor_agent import ( TextBeliefExtractorAgent as TextBeliefExtractorAgent, ) diff --git a/src/control_backend/agents/bdi/agentspeak_ast.py b/src/control_backend/agents/bdi/agentspeak_ast.py index 188b4f3..68be531 100644 --- a/src/control_backend/agents/bdi/agentspeak_ast.py +++ b/src/control_backend/agents/bdi/agentspeak_ast.py @@ -77,7 +77,7 @@ class AstTerm(AstExpression, ABC): return AstBinaryOp(self, BinaryOperatorType.NOT_EQUALS, _coalesce_expr(other)) -@dataclass +@dataclass(eq=False) class AstAtom(AstTerm): """ Grounded expression in all lowercase. @@ -89,7 +89,7 @@ class AstAtom(AstTerm): return self.value.lower() -@dataclass +@dataclass(eq=False) class AstVar(AstTerm): """ Ungrounded variable expression. First letter capitalized. @@ -101,7 +101,7 @@ class AstVar(AstTerm): return self.name.capitalize() -@dataclass +@dataclass(eq=False) class AstNumber(AstTerm): value: int | float @@ -109,7 +109,7 @@ class AstNumber(AstTerm): return str(self.value) -@dataclass +@dataclass(eq=False) class AstString(AstTerm): value: str @@ -117,7 +117,7 @@ class AstString(AstTerm): return f'"{self.value}"' -@dataclass +@dataclass(eq=False) class AstLiteral(AstTerm): functor: str terms: list[AstTerm] = field(default_factory=list) diff --git a/src/control_backend/agents/bdi/belief_collector_agent.py b/src/control_backend/agents/bdi/belief_collector_agent.py deleted file mode 100644 index ac0e2e5..0000000 --- a/src/control_backend/agents/bdi/belief_collector_agent.py +++ /dev/null @@ -1,152 +0,0 @@ -import json - -from pydantic import ValidationError - -from control_backend.agents.base import BaseAgent -from control_backend.core.agent_system import InternalMessage -from control_backend.core.config import settings -from control_backend.schemas.belief_message import Belief, BeliefMessage - - -class BDIBeliefCollectorAgent(BaseAgent): - """ - BDI Belief Collector Agent. - - This agent acts as a central aggregator for beliefs derived from various sources (e.g., text, - emotion, vision). It receives raw extracted data from other agents, - normalizes them into valid :class:`Belief` objects, and forwards them as a unified packet to the - BDI Core Agent. - - It serves as a funnel to ensure the BDI agent receives a consistent stream of beliefs. - """ - - async def setup(self): - """ - Initialize the agent. - """ - self.logger.info("Setting up %s", self.name) - - async def handle_message(self, msg: InternalMessage): - """ - Handle incoming messages from other extractor agents. - - Routes the message to specific handlers based on the 'type' field in the JSON body. - Supported types: - - ``belief_extraction_text``: Handled by :meth:`_handle_belief_text` - - ``emotion_extraction_text``: Handled by :meth:`_handle_emo_text` - - :param msg: The received internal message. - """ - sender_node = msg.sender - - # Parse JSON payload - try: - payload = json.loads(msg.body) - except Exception as e: - self.logger.warning( - "BeliefCollector: failed to parse JSON from %s. Body=%r Error=%s", - sender_node, - msg.body, - e, - ) - return - - msg_type = payload.get("type") - - # Prefer explicit 'type' field - if msg_type == "belief_extraction_text": - self.logger.debug("Message routed to _handle_belief_text (sender=%s)", sender_node) - await self._handle_belief_text(payload, sender_node) - # This is not implemented yet, but we keep the structure for future use - elif msg_type == "emotion_extraction_text": - self.logger.debug("Message routed to _handle_emo_text (sender=%s)", sender_node) - await self._handle_emo_text(payload, sender_node) - else: - self.logger.warning( - "Unrecognized message (sender=%s, type=%r). Ignoring.", sender_node, msg_type - ) - - async def _handle_belief_text(self, payload: dict, origin: str): - """ - Process text-based belief extraction payloads. - - Expected payload format:: - - { - "type": "belief_extraction_text", - "beliefs": { - "user_said": ["Can you help me?"], - "intention": ["ask_help"] - } - } - - Validates and converts the dictionary items into :class:`Belief` objects. - - :param payload: The dictionary payload containing belief data. - :param origin: The name of the sender agent. - """ - beliefs = payload.get("beliefs", {}) - - if not beliefs: - self.logger.debug("Received empty beliefs set.") - return - - def try_create_belief(name, arguments) -> Belief | None: - """ - Create a belief object from name and arguments, or return None silently if the input is - not correct. - - :param name: The name of the belief. - :param arguments: The arguments of the belief. - :return: A Belief object if the input is valid or None. - """ - try: - return Belief(name=name, arguments=arguments) - except ValidationError: - return None - - beliefs = [ - belief - for name, arguments in beliefs.items() - if (belief := try_create_belief(name, arguments)) is not None - ] - - self.logger.debug("Forwarding %d beliefs.", len(beliefs)) - for belief in beliefs: - for argument in belief.arguments: - self.logger.debug(" - %s %s", belief.name, argument) - - await self._send_beliefs_to_bdi(beliefs, origin=origin) - - async def _handle_emo_text(self, payload: dict, origin: str): - """ - Process emotion extraction payloads. - - **TODO**: Implement this method once emotion recognition is integrated. - - :param payload: The dictionary payload containing emotion data. - :param origin: The name of the sender agent. - """ - pass - - async def _send_beliefs_to_bdi(self, beliefs: list[Belief], origin: str | None = None): - """ - Send a list of aggregated beliefs to the BDI Core Agent. - - Wraps the beliefs in a :class:`BeliefMessage` and sends it via the 'beliefs' thread. - - :param beliefs: The list of Belief objects to send. - :param origin: (Optional) The original source of the beliefs (unused currently). - """ - if not beliefs: - return - - msg = InternalMessage( - to=settings.agent_settings.bdi_core_name, - sender=self.name, - body=BeliefMessage(create=beliefs).model_dump_json(), - thread="beliefs", - ) - - await self.send(msg) - self.logger.info("Sent %d belief(s) to BDI core.", len(beliefs)) diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 719053c..252502d 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -324,7 +324,7 @@ class RICommunicationAgent(BaseAgent): async def handle_message(self, msg: InternalMessage): try: pause_command = PauseCommand.model_validate_json(msg.body) - self._req_socket.send_json(pause_command.model_dump()) - self.logger.debug(self._req_socket.recv_json()) + await self._req_socket.send_json(pause_command.model_dump()) + self.logger.debug(await self._req_socket.recv_json()) except ValidationError: self.logger.warning("Incorrect message format for PauseCommand.") diff --git a/src/control_backend/main.py b/src/control_backend/main.py index d20cc66..ec93b1e 100644 --- a/src/control_backend/main.py +++ b/src/control_backend/main.py @@ -172,6 +172,8 @@ async def lifespan(app: FastAPI): await endpoints_pub_socket.send_multipart([PROGRAM_STATUS, ProgramStatus.STOPPING.value]) # Additional shutdown logic goes here + for agent in agents: + await agent.stop() logger.info("Application shutdown complete.") diff --git a/test/unit/agents/actuation/test_robot_gesture_agent.py b/test/unit/agents/actuation/test_robot_gesture_agent.py index fe051a6..225278d 100644 --- a/test/unit/agents/actuation/test_robot_gesture_agent.py +++ b/test/unit/agents/actuation/test_robot_gesture_agent.py @@ -28,7 +28,11 @@ async def test_setup_bind(zmq_context, mocker): settings = mocker.patch("control_backend.agents.actuation.robot_gesture_agent.settings") settings.zmq_settings.internal_sub_address = "tcp://internal:1234" - agent.add_behavior = MagicMock() + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) await agent.setup() @@ -55,7 +59,11 @@ async def test_setup_connect(zmq_context, mocker): settings = mocker.patch("control_backend.agents.actuation.robot_gesture_agent.settings") settings.zmq_settings.internal_sub_address = "tcp://internal:1234" - agent.add_behavior = MagicMock() + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) await agent.setup() @@ -119,6 +127,65 @@ async def test_handle_message_rejects_invalid_gesture_tag(): pubsocket.send_json.assert_not_awaited() +@pytest.mark.asyncio +async def test_handle_message_sends_valid_single_gesture_command(): + """Internal message with valid single gesture is forwarded.""" + pubsocket = AsyncMock() + agent = RobotGestureAgent("robot_gesture", single_gesture_data=["wave", "point"], address="") + agent.pubsocket = pubsocket + + payload = { + "endpoint": RIEndpoint.GESTURE_SINGLE, + "data": "wave", + } + msg = InternalMessage(to="robot", sender="tester", body=json.dumps(payload)) + + await agent.handle_message(msg) + + pubsocket.send_json.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_handle_message_rejects_invalid_single_gesture(): + """Internal message with invalid single gesture is not forwarded.""" + pubsocket = AsyncMock() + agent = RobotGestureAgent("robot_gesture", single_gesture_data=["wave", "point"], address="") + agent.pubsocket = pubsocket + + payload = { + "endpoint": RIEndpoint.GESTURE_SINGLE, + "data": "dance", + } + msg = InternalMessage(to="robot", sender="tester", body=json.dumps(payload)) + + await agent.handle_message(msg) + + pubsocket.send_json.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_zmq_command_loop_valid_single_gesture_payload(): + """UI command with valid single gesture is read from SUB and published.""" + command = {"endpoint": RIEndpoint.GESTURE_SINGLE, "data": "wave"} + fake_socket = AsyncMock() + + async def recv_once(): + agent._running = False + return b"command", json.dumps(command).encode("utf-8") + + fake_socket.recv_multipart = recv_once + fake_socket.send_json = AsyncMock() + + agent = RobotGestureAgent("robot_gesture", single_gesture_data=["wave", "point"], address="") + agent.subsocket = fake_socket + agent.pubsocket = fake_socket + agent._running = True + + await agent._zmq_command_loop() + + fake_socket.send_json.assert_awaited_once() + + @pytest.mark.asyncio async def test_handle_message_invalid_payload(): """Invalid payload is caught and does not send.""" diff --git a/test/unit/agents/actuation/test_robot_speech_agent.py b/test/unit/agents/actuation/test_robot_speech_agent.py index d95f66a..e5a664d 100644 --- a/test/unit/agents/actuation/test_robot_speech_agent.py +++ b/test/unit/agents/actuation/test_robot_speech_agent.py @@ -30,7 +30,11 @@ async def test_setup_bind(zmq_context, mocker): settings = mocker.patch("control_backend.agents.actuation.robot_speech_agent.settings") settings.zmq_settings.internal_sub_address = "tcp://internal:1234" - agent.add_behavior = MagicMock() + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) await agent.setup() @@ -48,7 +52,11 @@ async def test_setup_connect(zmq_context, mocker): settings = mocker.patch("control_backend.agents.actuation.robot_speech_agent.settings") settings.zmq_settings.internal_sub_address = "tcp://internal:1234" - agent.add_behavior = MagicMock() + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) await agent.setup() diff --git a/test/unit/agents/bdi/test_agentspeak_ast.py b/test/unit/agents/bdi/test_agentspeak_ast.py new file mode 100644 index 0000000..8d3bdf0 --- /dev/null +++ b/test/unit/agents/bdi/test_agentspeak_ast.py @@ -0,0 +1,186 @@ +import pytest + +from control_backend.agents.bdi.agentspeak_ast import ( + AstAtom, + AstBinaryOp, + AstLiteral, + AstLogicalExpression, + AstNumber, + AstPlan, + AstProgram, + AstRule, + AstStatement, + AstString, + AstVar, + BinaryOperatorType, + StatementType, + TriggerType, + _coalesce_expr, +) + + +def test_ast_atom(): + atom = AstAtom("test") + assert str(atom) == "test" + assert atom._to_agentspeak() == "test" + + +def test_ast_var(): + var = AstVar("Variable") + assert str(var) == "Variable" + assert var._to_agentspeak() == "Variable" + + +def test_ast_number(): + num = AstNumber(42) + assert str(num) == "42" + num_float = AstNumber(3.14) + assert str(num_float) == "3.14" + + +def test_ast_string(): + s = AstString("hello") + assert str(s) == '"hello"' + + +def test_ast_literal(): + lit = AstLiteral("functor", [AstAtom("atom"), AstNumber(1)]) + assert str(lit) == "functor(atom, 1)" + lit_empty = AstLiteral("functor") + assert str(lit_empty) == "functor" + + +def test_ast_binary_op(): + left = AstNumber(1) + right = AstNumber(2) + op = AstBinaryOp(left, BinaryOperatorType.GREATER_THAN, right) + assert str(op) == "1 > 2" + + # Test logical wrapper + assert isinstance(op.left, AstLogicalExpression) + assert isinstance(op.right, AstLogicalExpression) + + +def test_ast_binary_op_parens(): + # 1 > 2 + inner = AstBinaryOp(AstNumber(1), BinaryOperatorType.GREATER_THAN, AstNumber(2)) + # (1 > 2) & 3 + outer = AstBinaryOp(inner, BinaryOperatorType.AND, AstNumber(3)) + assert str(outer) == "(1 > 2) & 3" + + # 3 & (1 > 2) + outer_right = AstBinaryOp(AstNumber(3), BinaryOperatorType.AND, inner) + assert str(outer_right) == "3 & (1 > 2)" + + +def test_ast_binary_op_parens_negated(): + inner = AstLogicalExpression(AstAtom("foo"), negated=True) + outer = AstBinaryOp(inner, BinaryOperatorType.AND, AstAtom("bar")) + # The current implementation checks `if self.left.negated: l_str = f"({l_str})"` + # str(inner) is "not foo" + # so we expect "(not foo) & bar" + assert str(outer) == "(not foo) & bar" + + outer_right = AstBinaryOp(AstAtom("bar"), BinaryOperatorType.AND, inner) + assert str(outer_right) == "bar & (not foo)" + + +def test_ast_logical_expression_negation(): + expr = AstLogicalExpression(AstAtom("true"), negated=True) + assert str(expr) == "not true" + + expr_neg_neg = ~expr + assert str(expr_neg_neg) == "true" + assert not expr_neg_neg.negated + + # Invert a non-logical expression (wraps it) + term = AstAtom("true") + inverted = ~term + assert isinstance(inverted, AstLogicalExpression) + assert inverted.negated + assert str(inverted) == "not true" + + +def test_ast_logical_expression_no_negation(): + # _as_logical on already logical expression + expr = AstLogicalExpression(AstAtom("x")) + # Doing binary op will call _as_logical + op = AstBinaryOp(expr, BinaryOperatorType.AND, AstAtom("y")) + assert isinstance(op.left, AstLogicalExpression) + assert op.left is expr # Should reuse instance + + +def test_ast_operators(): + t1 = AstAtom("a") + t2 = AstAtom("b") + + assert str(t1 & t2) == "a & b" + assert str(t1 | t2) == "a | b" + assert str(t1 >= t2) == "a >= b" + assert str(t1 > t2) == "a > b" + assert str(t1 <= t2) == "a <= b" + assert str(t1 < t2) == "a < b" + assert str(t1 == t2) == "a == b" + assert str(t1 != t2) == r"a \== b" + + +def test_coalesce_expr(): + t = AstAtom("a") + assert str(t & "b") == 'a & "b"' + assert str(t & 1) == "a & 1" + assert str(t & 1.5) == "a & 1.5" + + with pytest.raises(TypeError): + _coalesce_expr(None) + + +def test_ast_statement(): + stmt = AstStatement(StatementType.DO_ACTION, AstLiteral("action")) + assert str(stmt) == ".action" + + +def test_ast_rule(): + # Rule with condition + rule = AstRule(AstLiteral("head"), AstLiteral("body")) + assert str(rule) == "head :- body." + + # Rule without condition + rule_simple = AstRule(AstLiteral("fact")) + assert str(rule_simple) == "fact." + + +def test_ast_plan(): + plan = AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("goal"), + [AstLiteral("context")], + [AstStatement(StatementType.DO_ACTION, AstLiteral("action"))], + ) + output = str(plan) + # verify parts exist + assert "+!goal" in output + assert ": context" in output + assert "<- .action." in output + + +def test_ast_plan_no_context(): + plan = AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("goal"), + [], + [AstStatement(StatementType.DO_ACTION, AstLiteral("action"))], + ) + output = str(plan) + assert "+!goal" in output + assert ": " not in output + assert "<- .action." in output + + +def test_ast_program(): + prog = AstProgram( + rules=[AstRule(AstLiteral("fact"))], + plans=[AstPlan(TriggerType.ADDED_BELIEF, AstLiteral("b"), [], [])], + ) + output = str(prog) + assert "fact." in output + assert "+b" in output diff --git a/test/unit/agents/bdi/test_agentspeak_generator.py b/test/unit/agents/bdi/test_agentspeak_generator.py new file mode 100644 index 0000000..5a3a849 --- /dev/null +++ b/test/unit/agents/bdi/test_agentspeak_generator.py @@ -0,0 +1,187 @@ +import uuid + +import pytest + +from control_backend.agents.bdi.agentspeak_ast import AstProgram +from control_backend.agents.bdi.agentspeak_generator import AgentSpeakGenerator +from control_backend.schemas.program import ( + BasicNorm, + ConditionalNorm, + Gesture, + GestureAction, + Goal, + InferredBelief, + KeywordBelief, + LLMAction, + LogicalOperator, + Phase, + Plan, + Program, + SemanticBelief, + SpeechAction, + Trigger, +) + + +@pytest.fixture +def generator(): + return AgentSpeakGenerator() + + +def test_generate_empty_program(generator): + prog = Program(phases=[]) + code = generator.generate(prog) + assert 'phase("end").' in code + assert "!notify_cycle" in code + + +def test_generate_basic_norm(generator): + norm = BasicNorm(id=uuid.uuid4(), name="n1", norm="be nice") + phase = Phase(id=uuid.uuid4(), norms=[norm], goals=[], triggers=[]) + prog = Program(phases=[phase]) + + code = generator.generate(prog) + assert f'norm("be nice") :- phase("{phase.id}").' in code + + +def test_generate_critical_norm(generator): + norm = BasicNorm(id=uuid.uuid4(), name="n1", norm="safety", critical=True) + phase = Phase(id=uuid.uuid4(), norms=[norm], goals=[], triggers=[]) + prog = Program(phases=[phase]) + + code = generator.generate(prog) + assert f'critical_norm("safety") :- phase("{phase.id}").' in code + + +def test_generate_conditional_norm(generator): + cond = KeywordBelief(id=uuid.uuid4(), name="k1", keyword="please") + norm = ConditionalNorm(id=uuid.uuid4(), name="n1", norm="help", condition=cond) + phase = Phase(id=uuid.uuid4(), norms=[norm], goals=[], triggers=[]) + prog = Program(phases=[phase]) + + code = generator.generate(prog) + assert 'norm("help")' in code + assert 'keyword_said("please")' in code + assert f"force_norm_{generator._slugify_str(norm.norm)}" in code + + +def test_generate_goal_and_plan(generator): + action = SpeechAction(id=uuid.uuid4(), name="s1", text="hello") + plan = Plan(id=uuid.uuid4(), name="p1", steps=[action]) + # IMPORTANT: can_fail must be False for +achieved_ belief to be added + goal = Goal(id=uuid.uuid4(), name="g1", description="desc", plan=plan, can_fail=False) + phase = Phase(id=uuid.uuid4(), norms=[], goals=[goal], triggers=[]) + prog = Program(phases=[phase]) + + code = generator.generate(prog) + # Check trigger for goal + goal_slug = generator._slugify_str(goal.name) + assert f"+!{goal_slug}" in code + assert f'phase("{phase.id}")' in code + assert '!say("hello")' in code + + # Check success belief addition + assert f"+achieved_{goal_slug}" in code + + +def test_generate_subgoal(generator): + subplan = Plan(id=uuid.uuid4(), name="p2", steps=[]) + subgoal = Goal(id=uuid.uuid4(), name="sub1", description="sub", plan=subplan) + + plan = Plan(id=uuid.uuid4(), name="p1", steps=[subgoal]) + goal = Goal(id=uuid.uuid4(), name="g1", description="main", plan=plan) + phase = Phase(id=uuid.uuid4(), norms=[], goals=[goal], triggers=[]) + prog = Program(phases=[phase]) + + code = generator.generate(prog) + subgoal_slug = generator._slugify_str(subgoal.name) + # Main goal calls subgoal + assert f"!{subgoal_slug}" in code + # Subgoal plan exists + assert f"+!{subgoal_slug}" in code + + +def test_generate_trigger(generator): + cond = SemanticBelief(id=uuid.uuid4(), name="s1", description="desc") + plan = Plan(id=uuid.uuid4(), name="p1", steps=[]) + trigger = Trigger(id=uuid.uuid4(), name="t1", condition=cond, plan=plan) + phase = Phase(id=uuid.uuid4(), norms=[], goals=[], triggers=[trigger]) + prog = Program(phases=[phase]) + + code = generator.generate(prog) + # Trigger logic is added to check_triggers + assert f"{generator.slugify(cond)}" in code + assert f'notify_trigger_start("{generator.slugify(trigger)}")' in code + assert f'notify_trigger_end("{generator.slugify(trigger)}")' in code + + +def test_phase_transition(generator): + phase1 = Phase(id=uuid.uuid4(), name="p1", norms=[], goals=[], triggers=[]) + phase2 = Phase(id=uuid.uuid4(), name="p2", norms=[], goals=[], triggers=[]) + prog = Program(phases=[phase1, phase2]) + + code = generator.generate(prog) + assert "transition_phase" in code + assert f'phase("{phase1.id}")' in code + assert f'phase("{phase2.id}")' in code + assert "force_transition_phase" in code + + +def test_astify_gesture(generator): + gesture = Gesture(type="single", name="wave") + action = GestureAction(id=uuid.uuid4(), name="g1", gesture=gesture) + ast = generator._astify(action) + assert str(ast) == 'gesture("single", "wave")' + + +def test_astify_llm_action(generator): + action = LLMAction(id=uuid.uuid4(), name="l1", goal="be funny") + ast = generator._astify(action) + assert str(ast) == 'reply_with_goal("be funny")' + + +def test_astify_inferred_belief_and(generator): + left = KeywordBelief(id=uuid.uuid4(), name="k1", keyword="a") + right = KeywordBelief(id=uuid.uuid4(), name="k2", keyword="b") + inf = InferredBelief( + id=uuid.uuid4(), name="i1", operator=LogicalOperator.AND, left=left, right=right + ) + + ast = generator._astify(inf) + assert 'keyword_said("a") & keyword_said("b")' == str(ast) + + +def test_astify_inferred_belief_or(generator): + left = KeywordBelief(id=uuid.uuid4(), name="k1", keyword="a") + right = KeywordBelief(id=uuid.uuid4(), name="k2", keyword="b") + inf = InferredBelief( + id=uuid.uuid4(), name="i1", operator=LogicalOperator.OR, left=left, right=right + ) + + ast = generator._astify(inf) + assert 'keyword_said("a") | keyword_said("b")' == str(ast) + + +def test_astify_semantic_belief(generator): + sb = SemanticBelief(id=uuid.uuid4(), name="s1", description="desc") + ast = generator._astify(sb) + assert str(ast) == f"semantic_{generator._slugify_str(sb.name)}" + + +def test_slugify_not_implemented(generator): + with pytest.raises(NotImplementedError): + generator.slugify("not a program element") + + +def test_astify_not_implemented(generator): + with pytest.raises(NotImplementedError): + generator._astify("not a program element") + + +def test_process_phase_transition_from_none(generator): + # Initialize AstProgram manually as we are bypassing generate() + generator._asp = AstProgram() + # Should safely return doing nothing + generator._add_phase_transition(None, None) + + assert len(generator._asp.plans) == 0 diff --git a/test/unit/agents/bdi/test_bdi_core_agent.py b/test/unit/agents/bdi/test_bdi_core_agent.py index 64f2ca7..152d901 100644 --- a/test/unit/agents/bdi/test_bdi_core_agent.py +++ b/test/unit/agents/bdi/test_bdi_core_agent.py @@ -57,11 +57,22 @@ async def test_handle_belief_collector_message(agent, mock_settings): await agent.handle_message(msg) - # Expect bdi_agent.call to be triggered to add belief - args = agent.bdi_agent.call.call_args.args - assert args[0] == agentspeak.Trigger.addition - assert args[1] == agentspeak.GoalType.belief - assert args[2] == agentspeak.Literal("user_said", (agentspeak.Literal("Hello"),)) + # Check for the specific call we expect among all calls + # bdi_agent.call is called multiple times (for transition_phase, check_triggers) + # We want to confirm the belief addition call exists + found_call = False + for call in agent.bdi_agent.call.call_args_list: + args = call.args + if ( + args[0] == agentspeak.Trigger.addition + and args[1] == agentspeak.GoalType.belief + and args[2].functor == "user_said" + and args[2].args[0].functor == "Hello" + ): + found_call = True + break + + assert found_call, "Expected belief addition call not found in bdi_agent.call history" @pytest.mark.asyncio @@ -77,11 +88,19 @@ async def test_handle_delete_belief_message(agent, mock_settings): ) await agent.handle_message(msg) - # Expect bdi_agent.call to be triggered to remove belief - args = agent.bdi_agent.call.call_args.args - assert args[0] == agentspeak.Trigger.removal - assert args[1] == agentspeak.GoalType.belief - assert args[2] == agentspeak.Literal("user_said", (agentspeak.Literal("Hello"),)) + found_call = False + for call in agent.bdi_agent.call.call_args_list: + args = call.args + if ( + args[0] == agentspeak.Trigger.removal + and args[1] == agentspeak.GoalType.belief + and args[2].functor == "user_said" + and args[2].args[0].functor == "Hello" + ): + found_call = True + break + + assert found_call @pytest.mark.asyncio @@ -171,7 +190,11 @@ def test_remove_belief_success_wakes_loop(agent): agent._remove_belief("remove_me", ["x"]) assert agent.bdi_agent.call.called - trigger, goaltype, literal, *_ = agent.bdi_agent.call.call_args.args + + call_args = agent.bdi_agent.call.call_args.args + trigger = call_args[0] + goaltype = call_args[1] + literal = call_args[2] assert trigger == agentspeak.Trigger.removal assert goaltype == agentspeak.GoalType.belief @@ -288,3 +311,216 @@ async def test_deadline_sleep_branch(agent): duration = time.time() - start_time assert duration >= 0.004 # loop slept until deadline + + +@pytest.mark.asyncio +async def test_handle_new_program(agent): + agent._load_asl = AsyncMock() + agent.add_behavior = MagicMock() + # Mock existing loop task so it can be cancelled + mock_task = MagicMock() + mock_task.cancel = MagicMock() + agent._bdi_loop_task = mock_task + + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) + + msg = InternalMessage(to="bdi_agent", thread="new_program", body="path/to/asl.asl") + + await agent.handle_message(msg) + + mock_task.cancel.assert_called_once() + agent._load_asl.assert_awaited_once_with("path/to/asl.asl") + agent.add_behavior.assert_called() + + +@pytest.mark.asyncio +async def test_handle_user_interrupts(agent, mock_settings): + mock_settings.agent_settings.user_interrupt_name = "user_interrupt_agent" + # force_phase_transition + agent._set_goal = MagicMock() + msg = InternalMessage( + to="bdi_agent", + sender=mock_settings.agent_settings.user_interrupt_name, + thread="force_phase_transition", + body="", + ) + await agent.handle_message(msg) + agent._set_goal.assert_called_with("transition_phase") + + # force_trigger + agent._force_trigger = MagicMock() + msg.thread = "force_trigger" + msg.body = "trigger_x" + await agent.handle_message(msg) + agent._force_trigger.assert_called_with("trigger_x") + + # force_norm + agent._force_norm = MagicMock() + msg.thread = "force_norm" + msg.body = "norm_y" + await agent.handle_message(msg) + agent._force_norm.assert_called_with("norm_y") + + # force_next_phase + agent._force_next_phase = MagicMock() + msg.thread = "force_next_phase" + msg.body = "" + await agent.handle_message(msg) + agent._force_next_phase.assert_called_once() + + # unknown interrupt + agent.logger = MagicMock() + msg.thread = "unknown_thing" + await agent.handle_message(msg) + agent.logger.warning.assert_called() + + +@pytest.mark.asyncio +async def test_custom_action_reply_with_goal(agent): + agent._send_to_llm = MagicMock(side_effect=agent.send) + agent._add_custom_actions() + action_fn = agent.actions.actions[(".reply_with_goal", 3)] + + mock_term = MagicMock(args=["msg", "norms", "goal"]) + gen = action_fn(agent, mock_term, MagicMock()) + next(gen) + agent._send_to_llm.assert_called_with("msg", "norms", "goal") + + +@pytest.mark.asyncio +async def test_custom_action_notify_norms(agent): + agent._add_custom_actions() + action_fn = agent.actions.actions[(".notify_norms", 1)] + + mock_term = MagicMock(args=["norms_list"]) + gen = action_fn(agent, mock_term, MagicMock()) + next(gen) + + agent.send.assert_called() + msg = agent.send.call_args[0][0] + assert msg.thread == "active_norms_update" + assert msg.body == "norms_list" + + +@pytest.mark.asyncio +async def test_custom_action_say(agent): + agent._add_custom_actions() + action_fn = agent.actions.actions[(".say", 1)] + + mock_term = MagicMock(args=["hello"]) + gen = action_fn(agent, mock_term, MagicMock()) + next(gen) + + assert agent.send.call_count == 2 + msgs = [c[0][0] for c in agent.send.call_args_list] + assert any(m.to == settings.agent_settings.robot_speech_name for m in msgs) + assert any( + m.to == settings.agent_settings.llm_name and m.thread == "assistant_message" for m in msgs + ) + + +@pytest.mark.asyncio +async def test_custom_action_gesture(agent): + agent._add_custom_actions() + # Test single + action_fn = agent.actions.actions[(".gesture", 2)] + mock_term = MagicMock(args=["single", "wave"]) + gen = action_fn(agent, mock_term, MagicMock()) + next(gen) + msg = agent.send.call_args[0][0] + assert "actuate/gesture/single" in msg.body + + # Test tag + mock_term.args = ["tag", "happy"] + gen = action_fn(agent, mock_term, MagicMock()) + next(gen) + msg = agent.send.call_args[0][0] + assert "actuate/gesture/tag" in msg.body + + +@pytest.mark.asyncio +async def test_custom_action_notify_user_said(agent): + agent._add_custom_actions() + action_fn = agent.actions.actions[(".notify_user_said", 1)] + mock_term = MagicMock(args=["hello"]) + gen = action_fn(agent, mock_term, MagicMock()) + next(gen) + msg = agent.send.call_args[0][0] + assert msg.to == settings.agent_settings.llm_name + assert msg.thread == "user_message" + + +@pytest.mark.asyncio +async def test_custom_action_notify_trigger_start_end(agent): + agent._add_custom_actions() + # Start + action_fn = agent.actions.actions[(".notify_trigger_start", 1)] + gen = action_fn(agent, MagicMock(args=["t1"]), MagicMock()) + next(gen) + assert agent.send.call_args[0][0].thread == "trigger_start" + + # End + action_fn = agent.actions.actions[(".notify_trigger_end", 1)] + gen = action_fn(agent, MagicMock(args=["t1"]), MagicMock()) + next(gen) + assert agent.send.call_args[0][0].thread == "trigger_end" + + +@pytest.mark.asyncio +async def test_custom_action_notify_goal_start(agent): + agent._add_custom_actions() + action_fn = agent.actions.actions[(".notify_goal_start", 1)] + gen = action_fn(agent, MagicMock(args=["g1"]), MagicMock()) + next(gen) + assert agent.send.call_args[0][0].thread == "goal_start" + + +@pytest.mark.asyncio +async def test_custom_action_notify_transition_phase(agent): + agent._add_custom_actions() + action_fn = agent.actions.actions[(".notify_transition_phase", 2)] + gen = action_fn(agent, MagicMock(args=["old", "new"]), MagicMock()) + next(gen) + msg = agent.send.call_args[0][0] + assert msg.thread == "transition_phase" + assert "old" in msg.body and "new" in msg.body + + +def test_remove_belief_no_args(agent): + agent._wake_bdi_loop = MagicMock() + agent.bdi_agent.call.return_value = True + agent._remove_belief("fact", None) + assert agent.bdi_agent.call.called + + +def test_set_goal_with_args(agent): + agent._wake_bdi_loop = MagicMock() + agent._set_goal("goal", ["arg1", "arg2"]) + assert agent.bdi_agent.call.called + + +def test_format_belief_string(): + assert BDICoreAgent.format_belief_string("b") == "b" + assert BDICoreAgent.format_belief_string("b", ["a1", "a2"]) == "b(a1,a2)" + + +def test_force_norm(agent): + agent._add_belief = MagicMock() + agent._force_norm("be_polite") + agent._add_belief.assert_called_with("force_be_polite") + + +def test_force_trigger(agent): + agent._set_goal = MagicMock() + agent._force_trigger("trig") + agent._set_goal.assert_called_with("trig") + + +def test_force_next_phase(agent): + agent._set_goal = MagicMock() + agent._force_next_phase() + agent._set_goal.assert_called_with("force_transition_phase") diff --git a/test/unit/agents/bdi/test_bdi_program_manager.py b/test/unit/agents/bdi/test_bdi_program_manager.py index 2bed2a7..540a172 100644 --- a/test/unit/agents/bdi/test_bdi_program_manager.py +++ b/test/unit/agents/bdi/test_bdi_program_manager.py @@ -1,13 +1,13 @@ import asyncio +import json import sys import uuid -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock, mock_open, patch import pytest from control_backend.agents.bdi.bdi_program_manager import BDIProgramManager from control_backend.core.agent_system import InternalMessage -from control_backend.schemas.belief_message import BeliefMessage from control_backend.schemas.program import BasicNorm, Goal, Phase, Plan, Program # Fix Windows Proactor loop for zmq @@ -48,24 +48,26 @@ def make_valid_program_json(norm="N1", goal="G1") -> str: ).model_dump_json() -@pytest.mark.skip(reason="Functionality being rebuilt.") @pytest.mark.asyncio -async def test_send_to_bdi(): +async def test_create_agentspeak_and_send_to_bdi(mock_settings): manager = BDIProgramManager(name="program_manager_test") manager.send = AsyncMock() program = Program.model_validate_json(make_valid_program_json()) - await manager._create_agentspeak_and_send_to_bdi(program) + + with patch("builtins.open", mock_open()) as mock_file: + await manager._create_agentspeak_and_send_to_bdi(program) + + # Check file writing + mock_file.assert_called_with("src/control_backend/agents/bdi/agentspeak.asl", "w") + handle = mock_file() + handle.write.assert_called() assert manager.send.await_count == 1 msg: InternalMessage = manager.send.await_args[0][0] - assert msg.thread == "beliefs" - - beliefs = BeliefMessage.model_validate_json(msg.body) - names = {b.name: b.arguments for b in beliefs.beliefs} - - assert "norms" in names and names["norms"] == ["N1"] - assert "goals" in names and names["goals"] == ["G1"] + assert msg.thread == "new_program" + assert msg.to == mock_settings.agent_settings.bdi_core_name + assert msg.body == "src/control_backend/agents/bdi/agentspeak.asl" @pytest.mark.asyncio @@ -81,6 +83,9 @@ async def test_receive_programs_valid_and_invalid(): manager.sub_socket = sub manager._create_agentspeak_and_send_to_bdi = AsyncMock() manager._send_clear_llm_history = AsyncMock() + manager._send_program_to_user_interrupt = AsyncMock() + manager._send_beliefs_to_semantic_belief_extractor = AsyncMock() + manager._send_goals_to_semantic_belief_extractor = AsyncMock() try: # Will give StopAsyncIteration when the predefined `sub.recv_multipart` side-effects run out @@ -94,10 +99,9 @@ async def test_receive_programs_valid_and_invalid(): assert forwarded.phases[0].norms[0].name == "N1" assert forwarded.phases[0].goals[0].name == "G1" - # Verify history clear was triggered - assert ( - manager._send_clear_llm_history.await_count == 2 - ) # first sends program to UserInterrupt, then clears LLM + # Verify history clear was triggered exactly once (for the valid program) + # The invalid program loop `continue`s before calling _send_clear_llm_history + assert manager._send_clear_llm_history.await_count == 1 @pytest.mark.asyncio @@ -115,4 +119,179 @@ async def test_send_clear_llm_history(mock_settings): # Verify the content and recipient assert msg.body == "clear_history" - assert msg.to == "llm_agent" + + +@pytest.mark.asyncio +async def test_handle_message_transition_phase(mock_settings): + mock_settings.agent_settings.user_interrupt_name = "user_interrupt_agent" + manager = BDIProgramManager(name="program_manager_test") + manager.send = AsyncMock() + + # Setup state + prog = Program.model_validate_json(make_valid_program_json(norm="N1", goal="G1")) + manager._initialize_internal_state(prog) + + # Test valid transition (to same phase for simplicity, or we need 2 phases) + # Let's create a program with 2 phases + phase2_id = uuid.uuid4() + phase2 = Phase(id=phase2_id, name="Phase 2", norms=[], goals=[], triggers=[]) + prog.phases.append(phase2) + manager._initialize_internal_state(prog) + + current_phase_id = str(prog.phases[0].id) + next_phase_id = str(phase2_id) + + payload = json.dumps({"old": current_phase_id, "new": next_phase_id}) + msg = InternalMessage(to="me", sender="bdi", body=payload, thread="transition_phase") + + await manager.handle_message(msg) + + assert str(manager._phase.id) == next_phase_id + + # Allow background tasks to run (add_behavior) + await asyncio.sleep(0) + + # Check notifications sent + # 1. beliefs to extractor + # 2. goals to extractor + # 3. notification to user interrupt + + assert manager.send.await_count >= 3 + + # Verify user interrupt notification + calls = manager.send.await_args_list + ui_msgs = [ + c[0][0] for c in calls if c[0][0].to == mock_settings.agent_settings.user_interrupt_name + ] + assert len(ui_msgs) > 0 + assert ui_msgs[-1].body == next_phase_id + + +@pytest.mark.asyncio +async def test_handle_message_transition_phase_desync(): + manager = BDIProgramManager(name="program_manager_test") + manager.logger = MagicMock() + + prog = Program.model_validate_json(make_valid_program_json()) + manager._initialize_internal_state(prog) + + current_phase_id = str(prog.phases[0].id) + + # Request transition from WRONG old phase + payload = json.dumps({"old": "wrong_id", "new": "some_new_id"}) + msg = InternalMessage(to="me", sender="bdi", body=payload, thread="transition_phase") + + await manager.handle_message(msg) + + # Should warn and do nothing + manager.logger.warning.assert_called_once() + assert "Phase transition desync detected" in manager.logger.warning.call_args[0][0] + assert str(manager._phase.id) == current_phase_id + + +@pytest.mark.asyncio +async def test_handle_message_transition_phase_end(mock_settings): + mock_settings.agent_settings.user_interrupt_name = "user_interrupt_agent" + manager = BDIProgramManager(name="program_manager_test") + manager.send = AsyncMock() + + prog = Program.model_validate_json(make_valid_program_json()) + manager._initialize_internal_state(prog) + current_phase_id = str(prog.phases[0].id) + + payload = json.dumps({"old": current_phase_id, "new": "end"}) + msg = InternalMessage(to="me", sender="bdi", body=payload, thread="transition_phase") + + await manager.handle_message(msg) + + assert manager._phase is None + + # Allow background tasks to run (add_behavior) + await asyncio.sleep(0) + + # Verify notification to user interrupt + assert manager.send.await_count == 1 + msg_sent = manager.send.await_args[0][0] + assert msg_sent.to == mock_settings.agent_settings.user_interrupt_name + assert msg_sent.body == "end" + + +@pytest.mark.asyncio +async def test_handle_message_achieve_goal(mock_settings): + mock_settings.agent_settings.text_belief_extractor_name = "text_belief_extractor_agent" + manager = BDIProgramManager(name="program_manager_test") + manager.send = AsyncMock() + + prog = Program.model_validate_json(make_valid_program_json(goal="TargetGoal")) + manager._initialize_internal_state(prog) + + goal_id = str(prog.phases[0].goals[0].id) + + msg = InternalMessage(to="me", sender="ui", body=goal_id, thread="achieve_goal") + + await manager.handle_message(msg) + + # Should send achieved goals to text extractor + assert manager.send.await_count == 1 + msg_sent = manager.send.await_args[0][0] + assert msg_sent.to == mock_settings.agent_settings.text_belief_extractor_name + assert msg_sent.thread == "achieved_goals" + + # Verify body + from control_backend.schemas.belief_list import GoalList + + gl = GoalList.model_validate_json(msg_sent.body) + assert len(gl.goals) == 1 + assert gl.goals[0].name == "TargetGoal" + + +@pytest.mark.asyncio +async def test_handle_message_achieve_goal_not_found(): + manager = BDIProgramManager(name="program_manager_test") + manager.send = AsyncMock() + manager.logger = MagicMock() + + prog = Program.model_validate_json(make_valid_program_json()) + manager._initialize_internal_state(prog) + + msg = InternalMessage(to="me", sender="ui", body="non_existent_id", thread="achieve_goal") + + await manager.handle_message(msg) + + manager.send.assert_not_called() + manager.logger.debug.assert_called() + + +@pytest.mark.asyncio +async def test_setup(mock_settings): + manager = BDIProgramManager(name="program_manager_test") + manager.send = AsyncMock() + + def close_coro(coro): + coro.close() + return MagicMock() + + manager.add_behavior = MagicMock(side_effect=close_coro) + + mock_context = MagicMock() + mock_sub = MagicMock() + mock_context.socket.return_value = mock_sub + + with patch( + "control_backend.agents.bdi.bdi_program_manager.Context.instance", return_value=mock_context + ): + # We also need to mock file writing in _create_agentspeak_and_send_to_bdi + with patch("builtins.open", new_callable=MagicMock): + await manager.setup() + + # Check logic + # 1. Sends default empty program to BDI + assert manager.send.await_count == 1 + assert manager.send.await_args[0][0].to == mock_settings.agent_settings.bdi_core_name + + # 2. Connects SUB socket + mock_sub.connect.assert_called_with(mock_settings.zmq_settings.internal_sub_address) + mock_sub.subscribe.assert_called_with("program") + + # 3. Adds behavior + manager.add_behavior.assert_called() diff --git a/test/unit/agents/bdi/test_belief_collector.py b/test/unit/agents/bdi/test_belief_collector.py deleted file mode 100644 index 69db269..0000000 --- a/test/unit/agents/bdi/test_belief_collector.py +++ /dev/null @@ -1,135 +0,0 @@ -import json -from unittest.mock import AsyncMock - -import pytest - -from control_backend.agents.bdi import ( - BDIBeliefCollectorAgent, -) -from control_backend.core.agent_system import InternalMessage -from control_backend.core.config import settings -from control_backend.schemas.belief_message import Belief - - -@pytest.fixture -def agent(): - agent = BDIBeliefCollectorAgent("belief_collector_agent") - return agent - - -def make_msg(body: dict, sender: str = "sender"): - return InternalMessage(to="collector", sender=sender, body=json.dumps(body)) - - -@pytest.mark.asyncio -async def test_handle_message_routes_belief_text(agent, mocker): - """ - Test that when a message is received, _handle_belief_text is called with that message. - """ - payload = {"type": "belief_extraction_text", "beliefs": {"user_said": [["hi"]]}} - spy = mocker.patch.object(agent, "_handle_belief_text", new_callable=AsyncMock) - - await agent.handle_message(make_msg(payload)) - - spy.assert_awaited_once_with(payload, "sender") - - -@pytest.mark.asyncio -async def test_handle_message_routes_emotion(agent, mocker): - payload = {"type": "emotion_extraction_text"} - spy = mocker.patch.object(agent, "_handle_emo_text", new_callable=AsyncMock) - - await agent.handle_message(make_msg(payload)) - - spy.assert_awaited_once_with(payload, "sender") - - -@pytest.mark.asyncio -async def test_handle_message_bad_json(agent, mocker): - agent._handle_belief_text = AsyncMock() - bad_msg = InternalMessage(to="collector", sender="sender", body="not json") - - await agent.handle_message(bad_msg) - - agent._handle_belief_text.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_handle_belief_text_sends_when_beliefs_exist(agent, mocker): - payload = {"type": "belief_extraction_text", "beliefs": {"user_said": ["hello"]}} - spy = mocker.patch.object(agent, "_send_beliefs_to_bdi", new_callable=AsyncMock) - expected = [Belief(name="user_said", arguments=["hello"])] - - await agent._handle_belief_text(payload, "origin") - - spy.assert_awaited_once_with(expected, origin="origin") - - -@pytest.mark.asyncio -async def test_handle_belief_text_no_send_when_empty(agent, mocker): - payload = {"type": "belief_extraction_text", "beliefs": {}} - spy = mocker.patch.object(agent, "_send_beliefs_to_bdi", new_callable=AsyncMock) - - await agent._handle_belief_text(payload, "origin") - - spy.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_send_beliefs_to_bdi(agent): - agent.send = AsyncMock() - beliefs = [Belief(name="user_said", arguments=["hello", "world"])] - - await agent._send_beliefs_to_bdi(beliefs, origin="origin") - - agent.send.assert_awaited_once() - sent: InternalMessage = agent.send.call_args.args[0] - assert sent.to == settings.agent_settings.bdi_core_name - assert sent.thread == "beliefs" - assert json.loads(sent.body)["create"] == [belief.model_dump() for belief in beliefs] - - -@pytest.mark.asyncio -async def test_setup_executes(agent): - """Covers setup and asserts the agent has a name.""" - await agent.setup() - assert agent.name == "belief_collector_agent" # simple property assertion - - -@pytest.mark.asyncio -async def test_handle_message_unrecognized_type_executes(agent): - """Covers the else branch for unrecognized message type.""" - payload = {"type": "unknown_type"} - msg = make_msg(payload, sender="tester") - # Wrap send to ensure nothing is sent - agent.send = AsyncMock() - await agent.handle_message(msg) - # Assert no messages were sent - agent.send.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_handle_emo_text_executes(agent): - """Covers the _handle_emo_text method.""" - # The method does nothing, but we can assert it returns None - result = await agent._handle_emo_text({}, "origin") - assert result is None - - -@pytest.mark.asyncio -async def test_send_beliefs_to_bdi_empty_executes(agent): - """Covers early return when beliefs are empty.""" - agent.send = AsyncMock() - await agent._send_beliefs_to_bdi({}) - # Assert that nothing was sent - agent.send.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_handle_belief_text_invalid_returns_none(agent, mocker): - payload = {"type": "belief_extraction_text", "beliefs": {"user_said": "invalid-argument"}} - - result = await agent._handle_belief_text(payload, "origin") - - # The method itself returns None - assert result is None diff --git a/test/unit/agents/bdi/test_text_belief_extractor.py b/test/unit/agents/bdi/test_text_belief_extractor.py index 6782ba1..0d7dc00 100644 --- a/test/unit/agents/bdi/test_text_belief_extractor.py +++ b/test/unit/agents/bdi/test_text_belief_extractor.py @@ -14,6 +14,7 @@ from control_backend.schemas.belief_message import Belief as InternalBelief from control_backend.schemas.belief_message import BeliefMessage from control_backend.schemas.chat_history import ChatHistory, ChatMessage from control_backend.schemas.program import ( + BaseGoal, # Changed from Goal ConditionalNorm, KeywordBelief, LLMAction, @@ -28,7 +29,8 @@ from control_backend.schemas.program import ( @pytest.fixture def llm(): llm = TextBeliefExtractorAgent.LLM(MagicMock(), 4) - llm._query_llm = AsyncMock() + # We must ensure _query_llm returns a dictionary so iterating it doesn't fail + llm._query_llm = AsyncMock(return_value={}) return llm @@ -374,3 +376,155 @@ async def test_llm_failure_handling(agent, llm, sample_program): assert len(belief_changes.true) == 0 assert len(belief_changes.false) == 0 + + +def test_belief_state_bool(): + # Empty + bs = BeliefState() + assert not bs + + # True set + bs_true = BeliefState(true={InternalBelief(name="a", arguments=None)}) + assert bs_true + + # False set + bs_false = BeliefState(false={InternalBelief(name="a", arguments=None)}) + assert bs_false + + +@pytest.mark.asyncio +async def test_handle_beliefs_message_validation_error(agent, mock_settings): + # Invalid JSON + mock_settings.agent_settings.bdi_program_manager_name = "bdi_program_manager_agent" + msg = InternalMessage( + to="me", + sender=mock_settings.agent_settings.bdi_program_manager_name, + thread="beliefs", + body="invalid json", + ) + # Should log warning and return + agent.logger = MagicMock() + await agent.handle_message(msg) + agent.logger.warning.assert_called() + + # Invalid Model + msg.body = json.dumps({"beliefs": [{"invalid": "obj"}]}) + await agent.handle_message(msg) + agent.logger.warning.assert_called() + + +@pytest.mark.asyncio +async def test_handle_goals_message_validation_error(agent, mock_settings): + mock_settings.agent_settings.bdi_program_manager_name = "bdi_program_manager_agent" + msg = InternalMessage( + to="me", + sender=mock_settings.agent_settings.bdi_program_manager_name, + thread="goals", + body="invalid json", + ) + agent.logger = MagicMock() + await agent.handle_message(msg) + agent.logger.warning.assert_called() + + +@pytest.mark.asyncio +async def test_handle_goal_achieved_message_validation_error(agent, mock_settings): + mock_settings.agent_settings.bdi_program_manager_name = "bdi_program_manager_agent" + msg = InternalMessage( + to="me", + sender=mock_settings.agent_settings.bdi_program_manager_name, + thread="achieved_goals", + body="invalid json", + ) + agent.logger = MagicMock() + await agent.handle_message(msg) + agent.logger.warning.assert_called() + + +@pytest.mark.asyncio +async def test_goal_inferrer_infer_from_conversation(agent, llm): + # Setup goals + # Use BaseGoal object as typically received by the extractor + g1 = BaseGoal(id=uuid.uuid4(), name="g1", description="desc", can_fail=True) + + # Use real GoalAchievementInferrer + from control_backend.agents.bdi.text_belief_extractor_agent import GoalAchievementInferrer + + inferrer = GoalAchievementInferrer(llm) + inferrer.goals = {g1} + + # Mock LLM response + llm._query_llm.return_value = True + + completions = await inferrer.infer_from_conversation(ChatHistory(messages=[])) + assert completions + # slugify uses slugify library, hard to predict exact string without it, + # but we can check values + assert list(completions.values())[0] is True + + +def test_apply_conversation_message_limit(agent): + with patch("control_backend.agents.bdi.text_belief_extractor_agent.settings") as mock_s: + mock_s.behaviour_settings.conversation_history_length_limit = 2 + agent.conversation.messages = [] + + agent._apply_conversation_message(ChatMessage(role="user", content="1")) + agent._apply_conversation_message(ChatMessage(role="assistant", content="2")) + agent._apply_conversation_message(ChatMessage(role="user", content="3")) + + assert len(agent.conversation.messages) == 2 + assert agent.conversation.messages[0].content == "2" + assert agent.conversation.messages[1].content == "3" + + +@pytest.mark.asyncio +async def test_handle_program_manager_reset(agent): + with patch("control_backend.agents.bdi.text_belief_extractor_agent.settings") as mock_s: + mock_s.agent_settings.bdi_program_manager_name = "pm" + agent.conversation.messages = [ChatMessage(role="user", content="hi")] + agent.belief_inferrer.available_beliefs = [ + SemanticBelief(id=uuid.uuid4(), name="b", description="d") + ] + + msg = InternalMessage(to="me", sender="pm", thread="conversation_history", body="reset") + await agent.handle_message(msg) + + assert len(agent.conversation.messages) == 0 + assert len(agent.belief_inferrer.available_beliefs) == 0 + + +def test_split_into_chunks(): + from control_backend.agents.bdi.text_belief_extractor_agent import SemanticBeliefInferrer + + items = [1, 2, 3, 4, 5] + chunks = SemanticBeliefInferrer._split_into_chunks(items, 2) + assert len(chunks) == 2 + assert len(chunks[0]) + len(chunks[1]) == 5 + + +@pytest.mark.asyncio +async def test_infer_beliefs_call(agent, llm): + from control_backend.agents.bdi.text_belief_extractor_agent import SemanticBeliefInferrer + + inferrer = SemanticBeliefInferrer(llm) + sb = SemanticBelief(id=uuid.uuid4(), name="is_happy", description="User is happy") + + llm.query = AsyncMock(return_value={"is_happy": True}) + + res = await inferrer._infer_beliefs(ChatHistory(messages=[]), [sb]) + assert res == {"is_happy": True} + llm.query.assert_called_once() + + +@pytest.mark.asyncio +async def test_infer_goal_call(agent, llm): + from control_backend.agents.bdi.text_belief_extractor_agent import GoalAchievementInferrer + + inferrer = GoalAchievementInferrer(llm) + goal = BaseGoal(id=uuid.uuid4(), name="g1", description="d") + + llm.query = AsyncMock(return_value=True) + + res = await inferrer._infer_goal(ChatHistory(messages=[]), goal) + assert res is True + llm.query.assert_called_once() diff --git a/test/unit/agents/communication/test_ri_communication_agent.py b/test/unit/agents/communication/test_ri_communication_agent.py index 06d8766..a678907 100644 --- a/test/unit/agents/communication/test_ri_communication_agent.py +++ b/test/unit/agents/communication/test_ri_communication_agent.py @@ -4,6 +4,8 @@ from unittest.mock import ANY, AsyncMock, MagicMock, patch import pytest from control_backend.agents.communication.ri_communication_agent import RICommunicationAgent +from control_backend.core.agent_system import InternalMessage +from control_backend.schemas.ri_message import PauseCommand, RIEndpoint def speech_agent_path(): @@ -53,7 +55,11 @@ async def test_setup_success_connects_and_starts_robot(zmq_context): MockGesture.return_value.start = AsyncMock() agent = RICommunicationAgent("ri_comm", address="tcp://localhost:5555", bind=False) - agent.add_behavior = MagicMock() + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) await agent.setup() @@ -83,7 +89,11 @@ async def test_setup_binds_when_requested(zmq_context): agent = RICommunicationAgent("ri_comm", address="tcp://localhost:5555", bind=True) - agent.add_behavior = MagicMock() + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) with ( patch(speech_agent_path(), autospec=True) as MockSpeech, @@ -151,6 +161,7 @@ async def test_handle_negotiation_response_updates_req_socket(zmq_context): @pytest.mark.asyncio async def test_handle_disconnection_publishes_and_reconnects(): pub_socket = AsyncMock() + pub_socket.close = MagicMock() agent = RICommunicationAgent("ri_comm") agent.pub_socket = pub_socket agent.connected = True @@ -233,6 +244,25 @@ async def test_handle_negotiation_response_unhandled_id(): ) +@pytest.mark.asyncio +async def test_handle_negotiation_response_audio(zmq_context): + agent = RICommunicationAgent("ri_comm") + + with patch( + "control_backend.agents.communication.ri_communication_agent.VADAgent", autospec=True + ) as MockVAD: + MockVAD.return_value.start = AsyncMock() + + await agent._handle_negotiation_response( + {"data": [{"id": "audio", "port": 7000, "bind": False}]} + ) + + MockVAD.assert_called_once_with( + audio_in_address="tcp://localhost:7000", audio_in_bind=False + ) + MockVAD.return_value.start.assert_awaited_once() + + @pytest.mark.asyncio async def test_stop_closes_sockets(): req = MagicMock() @@ -323,6 +353,7 @@ async def test_listen_loop_generic_exception(): @pytest.mark.asyncio async def test_handle_disconnection_timeout(monkeypatch): pub = AsyncMock() + pub.close = MagicMock() pub.send_multipart = AsyncMock(side_effect=TimeoutError) agent = RICommunicationAgent("ri_comm") @@ -365,3 +396,38 @@ async def test_negotiate_req_socket_none_causes_retry(zmq_context): result = await agent._negotiate_connection(max_retries=1) assert result is False + + +@pytest.mark.asyncio +async def test_handle_message_pause_command(zmq_context): + """Test handle_message with a valid PauseCommand.""" + agent = RICommunicationAgent("ri_comm") + agent._req_socket = AsyncMock() + agent.logger = MagicMock() + + agent._req_socket.recv_json.return_value = {"status": "ok"} + + pause_cmd = PauseCommand(data=True) + msg = InternalMessage(to="ri_comm", sender="user_int", body=pause_cmd.model_dump_json()) + + await agent.handle_message(msg) + + agent._req_socket.send_json.assert_awaited_once() + args = agent._req_socket.send_json.await_args[0][0] + assert args["endpoint"] == RIEndpoint.PAUSE.value + assert args["data"] is True + + +@pytest.mark.asyncio +async def test_handle_message_invalid_pause_command(zmq_context): + """Test handle_message with invalid JSON.""" + agent = RICommunicationAgent("ri_comm") + agent._req_socket = AsyncMock() + agent.logger = MagicMock() + + msg = InternalMessage(to="ri_comm", sender="user_int", body="invalid json") + + await agent.handle_message(msg) + + agent.logger.warning.assert_called_with("Incorrect message format for PauseCommand.") + agent._req_socket.send_json.assert_not_called() diff --git a/test/unit/agents/llm/test_llm_agent.py b/test/unit/agents/llm/test_llm_agent.py index 5fc07f2..a1cc297 100644 --- a/test/unit/agents/llm/test_llm_agent.py +++ b/test/unit/agents/llm/test_llm_agent.py @@ -58,17 +58,20 @@ async def test_llm_processing_success(mock_httpx_client, mock_settings): to="llm_agent", sender=mock_settings.agent_settings.bdi_core_name, body=prompt.model_dump_json(), + thread="prompt_message", # REQUIRED: thread must match handle_message logic ) await agent.handle_message(msg) # Verification # "Hello world." constitutes one sentence/chunk based on punctuation split - # The agent should call send once with the full sentence + # The agent should call send once with the full sentence, PLUS once more for full reply assert agent.send.called - args = agent.send.call_args_list[0][0][0] - assert args.to == mock_settings.agent_settings.bdi_core_name - assert "Hello world." in args.body + + # Check args. We expect at least one call sending "Hello world." + calls = agent.send.call_args_list + bodies = [c[0][0].body for c in calls] + assert any("Hello world." in b for b in bodies) @pytest.mark.asyncio @@ -80,18 +83,23 @@ async def test_llm_processing_errors(mock_httpx_client, mock_settings): to="llm", sender=mock_settings.agent_settings.bdi_core_name, body=prompt.model_dump_json(), + thread="prompt_message", ) - # HTTP Error + # HTTP Error: stream method RAISES exception immediately mock_httpx_client.stream = MagicMock(side_effect=httpx.HTTPError("Fail")) + await agent.handle_message(msg) - assert "LLM service unavailable." in agent.send.call_args[0][0].body + + # Check that error message was sent + assert agent.send.called + assert "LLM service unavailable." in agent.send.call_args_list[0][0][0].body # General Exception agent.send.reset_mock() mock_httpx_client.stream = MagicMock(side_effect=Exception("Boom")) await agent.handle_message(msg) - assert "Error processing the request." in agent.send.call_args[0][0].body + assert "Error processing the request." in agent.send.call_args_list[0][0][0].body @pytest.mark.asyncio @@ -113,16 +121,19 @@ async def test_llm_json_error(mock_httpx_client, mock_settings): agent = LLMAgent("llm_agent") agent.send = AsyncMock() + # Ensure logger is mocked + agent.logger = MagicMock() - with patch.object(agent.logger, "error") as log: - prompt = LLMPromptMessage(text="Hi", norms=[], goals=[]) - msg = InternalMessage( - to="llm", - sender=mock_settings.agent_settings.bdi_core_name, - body=prompt.model_dump_json(), - ) - await agent.handle_message(msg) - log.assert_called() # Should log JSONDecodeError + prompt = LLMPromptMessage(text="Hi", norms=[], goals=[]) + msg = InternalMessage( + to="llm", + sender=mock_settings.agent_settings.bdi_core_name, + body=prompt.model_dump_json(), + thread="prompt_message", + ) + await agent.handle_message(msg) + + agent.logger.error.assert_called() # Should log JSONDecodeError def test_llm_instructions(): @@ -157,6 +168,7 @@ async def test_handle_message_validation_error_branch_no_send(mock_httpx_client, to="llm_agent", sender=mock_settings.agent_settings.bdi_core_name, body=invalid_json, + thread="prompt_message", ) await agent.handle_message(msg) @@ -285,3 +297,28 @@ async def test_clear_history_command(mock_settings): ) await agent.handle_message(msg) assert len(agent.history) == 0 + + +@pytest.mark.asyncio +async def test_handle_assistant_and_user_messages(mock_settings): + agent = LLMAgent("llm_agent") + + # Assistant message + msg_ast = InternalMessage( + to="llm_agent", + sender=mock_settings.agent_settings.bdi_core_name, + thread="assistant_message", + body="I said this", + ) + await agent.handle_message(msg_ast) + assert agent.history[-1] == {"role": "assistant", "content": "I said this"} + + # User message + msg_usr = InternalMessage( + to="llm_agent", + sender=mock_settings.agent_settings.bdi_core_name, + thread="user_message", + body="User said this", + ) + await agent.handle_message(msg_usr) + assert agent.history[-1] == {"role": "user", "content": "User said this"} diff --git a/test/unit/agents/perception/transcription_agent/test_transcription_agent.py b/test/unit/agents/perception/transcription_agent/test_transcription_agent.py index ccdaa7f..57875ca 100644 --- a/test/unit/agents/perception/transcription_agent/test_transcription_agent.py +++ b/test/unit/agents/perception/transcription_agent/test_transcription_agent.py @@ -36,7 +36,12 @@ async def test_transcription_agent_flow(mock_zmq_context): agent.send = AsyncMock() agent._running = True - agent.add_behavior = AsyncMock() + + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) await agent.setup() @@ -143,7 +148,12 @@ async def test_transcription_loop_continues_after_error(mock_zmq_context): agent = TranscriptionAgent("tcp://in") agent._running = True # ← REQUIRED to enter the loop agent.send = AsyncMock() # should never be called - agent.add_behavior = AsyncMock() # match other tests + + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) # match other tests await agent.setup() @@ -180,7 +190,12 @@ async def test_transcription_continue_branch_when_empty(mock_zmq_context): # Make loop runnable agent._running = True agent.send = AsyncMock() - agent.add_behavior = AsyncMock() + + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) await agent.setup() diff --git a/test/unit/agents/perception/vad_agent/test_vad_agent.py b/test/unit/agents/perception/vad_agent/test_vad_agent.py new file mode 100644 index 0000000..fe65545 --- /dev/null +++ b/test/unit/agents/perception/vad_agent/test_vad_agent.py @@ -0,0 +1,153 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from control_backend.agents.perception.vad_agent import VADAgent +from control_backend.core.agent_system import InternalMessage +from control_backend.schemas.program_status import PROGRAM_STATUS, ProgramStatus + + +@pytest.fixture(autouse=True) +def mock_zmq(): + with patch("zmq.asyncio.Context") as mock: + mock.instance.return_value = MagicMock() + yield mock + + +@pytest.fixture +def agent(): + return VADAgent("tcp://localhost:5555", False) + + +@pytest.mark.asyncio +async def test_handle_message_pause(agent): + agent._paused = MagicMock() + # It starts set (not paused) + + msg = InternalMessage(to="vad", sender="user_interrupt_agent", body="PAUSE") + + # We need to mock settings to match sender name + with patch("control_backend.agents.perception.vad_agent.settings") as mock_settings: + mock_settings.agent_settings.user_interrupt_name = "user_interrupt_agent" + + await agent.handle_message(msg) + + agent._paused.clear.assert_called_once() + assert agent._reset_needed is True + + +@pytest.mark.asyncio +async def test_handle_message_resume(agent): + agent._paused = MagicMock() + msg = InternalMessage(to="vad", sender="user_interrupt_agent", body="RESUME") + + with patch("control_backend.agents.perception.vad_agent.settings") as mock_settings: + mock_settings.agent_settings.user_interrupt_name = "user_interrupt_agent" + + await agent.handle_message(msg) + + agent._paused.set.assert_called_once() + + +@pytest.mark.asyncio +async def test_handle_message_unknown_command(agent): + agent._paused = MagicMock() + msg = InternalMessage(to="vad", sender="user_interrupt_agent", body="UNKNOWN") + + with patch("control_backend.agents.perception.vad_agent.settings") as mock_settings: + mock_settings.agent_settings.user_interrupt_name = "user_interrupt_agent" + agent.logger = MagicMock() + + await agent.handle_message(msg) + + agent.logger.warning.assert_called() + agent._paused.clear.assert_not_called() + agent._paused.set.assert_not_called() + + +@pytest.mark.asyncio +async def test_handle_message_unknown_sender(agent): + agent._paused = MagicMock() + msg = InternalMessage(to="vad", sender="other_agent", body="PAUSE") + + with patch("control_backend.agents.perception.vad_agent.settings") as mock_settings: + mock_settings.agent_settings.user_interrupt_name = "user_interrupt_agent" + + await agent.handle_message(msg) + + agent._paused.clear.assert_not_called() + + +@pytest.mark.asyncio +async def test_status_loop_waits_for_running(agent): + agent._running = True + agent.program_sub_socket = AsyncMock() + agent.program_sub_socket.close = MagicMock() + agent._reset_stream = AsyncMock() + + # Sequence of messages: + # 1. Wrong topic + # 2. Right topic, wrong status (STARTING) + # 3. Right topic, RUNNING -> Should break loop + + agent.program_sub_socket.recv_multipart.side_effect = [ + (b"wrong_topic", b"whatever"), + (PROGRAM_STATUS, ProgramStatus.STARTING.value), + (PROGRAM_STATUS, ProgramStatus.RUNNING.value), + ] + + await agent._status_loop() + + assert agent._reset_stream.await_count == 1 + agent.program_sub_socket.close.assert_called_once() + + +@pytest.mark.asyncio +async def test_setup_success(agent, mock_zmq): + def close_coro(coro): + coro.close() + return MagicMock() + + agent.add_behavior = MagicMock(side_effect=close_coro) + + mock_context = mock_zmq.instance.return_value + mock_sub = MagicMock() + mock_pub = MagicMock() + + # We expect multiple socket calls: + # 1. audio_in (SUB) + # 2. audio_out (PUB) + # 3. program_sub (SUB) + mock_context.socket.side_effect = [mock_sub, mock_pub, mock_sub] + + with patch("control_backend.agents.perception.vad_agent.torch.hub.load") as mock_load: + mock_load.return_value = (MagicMock(), None) + + with patch("control_backend.agents.perception.vad_agent.TranscriptionAgent") as MockTrans: + mock_trans_instance = MockTrans.return_value + mock_trans_instance.start = AsyncMock() + + await agent.setup() + + mock_trans_instance.start.assert_awaited_once() + + assert agent.add_behavior.call_count == 2 # streaming_loop + status_loop + assert agent.audio_in_socket is not None + assert agent.audio_out_socket is not None + assert agent.program_sub_socket is not None + + +@pytest.mark.asyncio +async def test_reset_stream(agent): + mock_poller = MagicMock() + agent.audio_in_poller = mock_poller + + # poll(1) returns not None twice, then None + mock_poller.poll = AsyncMock(side_effect=[b"data", b"data", None]) + + agent._ready = MagicMock() + + await agent._reset_stream() + + assert mock_poller.poll.await_count == 3 + agent._ready.set.assert_called_once() diff --git a/test/unit/agents/perception/vad_agent/test_vad_streaming.py b/test/unit/agents/perception/vad_agent/test_vad_streaming.py index 166919f..349fab2 100644 --- a/test/unit/agents/perception/vad_agent/test_vad_streaming.py +++ b/test/unit/agents/perception/vad_agent/test_vad_streaming.py @@ -5,6 +5,7 @@ import pytest import zmq from control_backend.agents.perception.vad_agent import VADAgent +from control_backend.core.config import settings # We don't want to use real ZMQ in unit tests, for example because it can give errors when sockets @@ -135,6 +136,54 @@ async def test_no_data(audio_out_socket, vad_agent): assert len(vad_agent.audio_buffer) == 0 +@pytest.mark.asyncio +async def test_streaming_loop_reset_needed(audio_out_socket, vad_agent): + """Test that _reset_needed branch works as expected.""" + vad_agent._reset_needed = True + vad_agent._ready.set() + vad_agent._paused.set() + vad_agent._running = True + vad_agent.audio_buffer = np.array([1.0], dtype=np.float32) + vad_agent.i_since_speech = 0 + + # Mock _reset_stream to stop the loop by setting _running=False + async def mock_reset(): + vad_agent._running = False + + vad_agent._reset_stream = mock_reset + + # Needs a poller to avoid AssertionError + vad_agent.audio_in_poller = AsyncMock() + vad_agent.audio_in_poller.poll.return_value = None + + await vad_agent._streaming_loop() + + assert vad_agent._reset_needed is False + assert len(vad_agent.audio_buffer) == 0 + assert vad_agent.i_since_speech == settings.behaviour_settings.vad_initial_since_speech + + +@pytest.mark.asyncio +async def test_streaming_loop_no_data_clears_buffer(audio_out_socket, vad_agent): + """Test that if poll returns None, buffer is cleared if not empty.""" + vad_agent.audio_buffer = np.array([1.0], dtype=np.float32) + vad_agent._ready.set() + vad_agent._paused.set() + vad_agent._running = True + + class MockPoller: + async def poll(self, timeout_ms=None): + vad_agent._running = False # stop after one poll + return None + + vad_agent.audio_in_poller = MockPoller() + + await vad_agent._streaming_loop() + + assert len(vad_agent.audio_buffer) == 0 + assert vad_agent.i_since_speech == settings.behaviour_settings.vad_initial_since_speech + + @pytest.mark.asyncio async def test_vad_model_load_failure_stops_agent(vad_agent): """ diff --git a/test/unit/agents/test_base.py b/test/unit/agents/test_base.py new file mode 100644 index 0000000..0579ada --- /dev/null +++ b/test/unit/agents/test_base.py @@ -0,0 +1,24 @@ +import logging + +from control_backend.agents.base import BaseAgent + + +class MyAgent(BaseAgent): + async def setup(self): + pass + + async def handle_message(self, msg): + pass + + +def test_base_agent_logger_init(): + # When defining a subclass, __init_subclass__ runs + # The BaseAgent in agents/base.py sets the logger + assert hasattr(MyAgent, "logger") + assert isinstance(MyAgent.logger, logging.Logger) + # The logger name depends on the package. + # Since this test file is running as a module, __package__ might be None or the test package. + # In 'src/control_backend/agents/base.py', it uses __package__ of base.py which is + # 'control_backend.agents'. + # So logger name should be control_backend.agents.MyAgent + assert MyAgent.logger.name == "control_backend.agents.MyAgent" diff --git a/test/unit/agents/user_interrupt/test_user_interrupt.py b/test/unit/agents/user_interrupt/test_user_interrupt.py index 7e3e700..7c38a05 100644 --- a/test/unit/agents/user_interrupt/test_user_interrupt.py +++ b/test/unit/agents/user_interrupt/test_user_interrupt.py @@ -7,6 +7,15 @@ import pytest from control_backend.agents.user_interrupt.user_interrupt_agent import UserInterruptAgent from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings +from control_backend.schemas.program import ( + ConditionalNorm, + Goal, + KeywordBelief, + Phase, + Plan, + Program, + Trigger, +) from control_backend.schemas.ri_message import RIEndpoint @@ -16,6 +25,7 @@ def agent(): agent.send = AsyncMock() agent.logger = MagicMock() agent.sub_socket = AsyncMock() + agent.pub_socket = AsyncMock() return agent @@ -49,21 +59,18 @@ async def test_send_to_gesture_agent(agent): @pytest.mark.asyncio -async def test_send_to_program_manager(agent): +async def test_send_to_bdi_belief(agent): """Verify belief update format.""" - context_str = "2" + context_str = "some_goal" - await agent._send_to_program_manager(context_str) + await agent._send_to_bdi_belief(context_str) - agent.send.assert_awaited_once() - sent_msg: InternalMessage = agent.send.call_args.args[0] + assert agent.send.await_count == 1 + sent_msg = agent.send.call_args.args[0] - assert sent_msg.to == settings.agent_settings.bdi_program_manager_name - assert sent_msg.thread == "belief_override_id" - - body = json.loads(sent_msg.body) - - assert body["belief"] == context_str + assert sent_msg.to == settings.agent_settings.bdi_core_name + assert sent_msg.thread == "beliefs" + assert "achieved_some_goal" in sent_msg.body @pytest.mark.asyncio @@ -77,6 +84,10 @@ async def test_receive_loop_routing_success(agent): # Prepare JSON payloads as bytes payload_speech = json.dumps({"type": "speech", "context": "Hello Speech"}).encode() payload_gesture = json.dumps({"type": "gesture", "context": "Hello Gesture"}).encode() + # override calls _send_to_bdi (for trigger/norm) OR _send_to_bdi_belief (for goal). + + # To test routing, we need to populate the maps + agent._goal_map["Hello Override"] = "some_goal_slug" payload_override = json.dumps({"type": "override", "context": "Hello Override"}).encode() agent.sub_socket.recv_multipart.side_effect = [ @@ -88,7 +99,7 @@ async def test_receive_loop_routing_success(agent): agent._send_to_speech_agent = AsyncMock() agent._send_to_gesture_agent = AsyncMock() - agent._send_to_program_manager = AsyncMock() + agent._send_to_bdi_belief = AsyncMock() try: await agent._receive_button_event() @@ -103,12 +114,12 @@ async def test_receive_loop_routing_success(agent): # Gesture agent._send_to_gesture_agent.assert_awaited_once_with("Hello Gesture") - # Override - agent._send_to_program_manager.assert_awaited_once_with("Hello Override") + # Override (since we mapped it to a goal) + agent._send_to_bdi_belief.assert_awaited_once_with("some_goal_slug") assert agent._send_to_speech_agent.await_count == 1 assert agent._send_to_gesture_agent.await_count == 1 - assert agent._send_to_program_manager.await_count == 1 + assert agent._send_to_bdi_belief.await_count == 1 @pytest.mark.asyncio @@ -125,7 +136,6 @@ async def test_receive_loop_unknown_type(agent): agent._send_to_speech_agent = AsyncMock() agent._send_to_gesture_agent = AsyncMock() - agent._send_to_belief_collector = AsyncMock() try: await agent._receive_button_event() @@ -137,10 +147,165 @@ async def test_receive_loop_unknown_type(agent): # Ensure no handlers were called agent._send_to_speech_agent.assert_not_called() agent._send_to_gesture_agent.assert_not_called() - agent._send_to_belief_collector.assert_not_called() - agent.logger.warning.assert_called_with( - "Received button press with unknown type '%s' (context: '%s').", - "unknown_thing", - "some_data", - ) + agent.logger.warning.assert_called() + + +@pytest.mark.asyncio +async def test_create_mapping(agent): + # Create a program with a trigger, goal, and conditional norm + import uuid + + trigger_id = uuid.uuid4() + goal_id = uuid.uuid4() + norm_id = uuid.uuid4() + + cond = KeywordBelief(id=uuid.uuid4(), name="k1", keyword="key") + plan = Plan(id=uuid.uuid4(), name="p1", steps=[]) + + trigger = Trigger(id=trigger_id, name="my_trigger", condition=cond, plan=plan) + goal = Goal(id=goal_id, name="my_goal", description="desc", plan=plan) + + cn = ConditionalNorm(id=norm_id, name="my_norm", norm="be polite", condition=cond) + + phase = Phase(id=uuid.uuid4(), name="phase1", norms=[cn], goals=[goal], triggers=[trigger]) + prog = Program(phases=[phase]) + + # Call create_mapping via handle_message + msg = InternalMessage(to="me", thread="new_program", body=prog.model_dump_json()) + await agent.handle_message(msg) + + # Check maps + assert str(trigger_id) in agent._trigger_map + assert agent._trigger_map[str(trigger_id)] == "trigger_my_trigger" + + assert str(goal_id) in agent._goal_map + assert agent._goal_map[str(goal_id)] == "my_goal" + + assert str(norm_id) in agent._cond_norm_map + assert agent._cond_norm_map[str(norm_id)] == "norm_be_polite" + + +@pytest.mark.asyncio +async def test_create_mapping_invalid_json(agent): + # Pass invalid json to handle_message thread "new_program" + msg = InternalMessage(to="me", thread="new_program", body="invalid json") + await agent.handle_message(msg) + + # Should log error and maps should remain empty or cleared + agent.logger.error.assert_called() + + +@pytest.mark.asyncio +async def test_handle_message_trigger_start(agent): + # Setup reverse map manually + agent._trigger_reverse_map["trigger_slug"] = "ui_id_123" + + msg = InternalMessage(to="me", thread="trigger_start", body="trigger_slug") + await agent.handle_message(msg) + + agent.pub_socket.send_multipart.assert_awaited_once() + args = agent.pub_socket.send_multipart.call_args[0][0] + assert args[0] == b"experiment" + payload = json.loads(args[1]) + assert payload["type"] == "trigger_update" + assert payload["id"] == "ui_id_123" + assert payload["achieved"] is True + + +@pytest.mark.asyncio +async def test_handle_message_trigger_end(agent): + agent._trigger_reverse_map["trigger_slug"] = "ui_id_123" + + msg = InternalMessage(to="me", thread="trigger_end", body="trigger_slug") + await agent.handle_message(msg) + + agent.pub_socket.send_multipart.assert_awaited_once() + payload = json.loads(agent.pub_socket.send_multipart.call_args[0][0][1]) + assert payload["type"] == "trigger_update" + assert payload["achieved"] is False + + +@pytest.mark.asyncio +async def test_handle_message_transition_phase(agent): + msg = InternalMessage(to="me", thread="transition_phase", body="phase_id_123") + await agent.handle_message(msg) + + agent.pub_socket.send_multipart.assert_awaited_once() + payload = json.loads(agent.pub_socket.send_multipart.call_args[0][0][1]) + assert payload["type"] == "phase_update" + assert payload["id"] == "phase_id_123" + + +@pytest.mark.asyncio +async def test_handle_message_goal_start(agent): + agent._goal_reverse_map["goal_slug"] = "goal_id_123" + + msg = InternalMessage(to="me", thread="goal_start", body="goal_slug") + await agent.handle_message(msg) + + agent.pub_socket.send_multipart.assert_awaited_once() + payload = json.loads(agent.pub_socket.send_multipart.call_args[0][0][1]) + assert payload["type"] == "goal_update" + assert payload["id"] == "goal_id_123" + assert payload["active"] is True + + +@pytest.mark.asyncio +async def test_handle_message_active_norms_update(agent): + agent._cond_norm_reverse_map["norm_active"] = "id_1" + agent._cond_norm_reverse_map["norm_inactive"] = "id_2" + + # Body is like: "('norm_active', 'other')" + # The split logic handles quotes etc. + msg = InternalMessage(to="me", thread="active_norms_update", body="'norm_active', 'other'") + await agent.handle_message(msg) + + agent.pub_socket.send_multipart.assert_awaited_once() + payload = json.loads(agent.pub_socket.send_multipart.call_args[0][0][1]) + assert payload["type"] == "cond_norms_state_update" + norms = {n["id"]: n["active"] for n in payload["norms"]} + assert norms["id_1"] is True + assert norms["id_2"] is False + + +@pytest.mark.asyncio +async def test_send_experiment_control(agent): + # Test next_phase + await agent._send_experiment_control_to_bdi_core("next_phase") + agent.send.assert_awaited() + msg = agent.send.call_args[0][0] + assert msg.thread == "force_next_phase" + + # Test reset_phase + await agent._send_experiment_control_to_bdi_core("reset_phase") + msg = agent.send.call_args[0][0] + assert msg.thread == "reset_current_phase" + + # Test reset_experiment + await agent._send_experiment_control_to_bdi_core("reset_experiment") + msg = agent.send.call_args[0][0] + assert msg.thread == "reset_experiment" + + +@pytest.mark.asyncio +async def test_send_pause_command(agent): + await agent._send_pause_command("true") + # Sends to RI and VAD + assert agent.send.await_count == 2 + msgs = [call.args[0] for call in agent.send.call_args_list] + + ri_msg = next(m for m in msgs if m.to == settings.agent_settings.ri_communication_name) + assert json.loads(ri_msg.body)["endpoint"] == "" # PAUSE endpoint + assert json.loads(ri_msg.body)["data"] is True + + vad_msg = next(m for m in msgs if m.to == settings.agent_settings.vad_name) + assert vad_msg.body == "PAUSE" + + agent.send.reset_mock() + await agent._send_pause_command("false") + assert agent.send.await_count == 2 + vad_msg = next( + m for m in agent.send.call_args_list if m.args[0].to == settings.agent_settings.vad_name + ).args[0] + assert vad_msg.body == "RESUME" diff --git a/test/unit/api/v1/endpoints/test_user_interact.py b/test/unit/api/v1/endpoints/test_user_interact.py new file mode 100644 index 0000000..ddb9932 --- /dev/null +++ b/test/unit/api/v1/endpoints/test_user_interact.py @@ -0,0 +1,96 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from control_backend.api.v1.endpoints import user_interact + + +@pytest.fixture +def app(): + app = FastAPI() + app.include_router(user_interact.router) + return app + + +@pytest.fixture +def client(app): + return TestClient(app) + + +@pytest.mark.asyncio +async def test_receive_button_event(client): + mock_pub_socket = AsyncMock() + client.app.state.endpoints_pub_socket = mock_pub_socket + + payload = {"type": "speech", "context": "hello"} + response = client.post("/button_pressed", json=payload) + + assert response.status_code == 202 + assert response.json() == {"status": "Event received"} + + mock_pub_socket.send_multipart.assert_awaited_once() + args = mock_pub_socket.send_multipart.call_args[0][0] + assert args[0] == b"button_pressed" + assert "speech" in args[1].decode() + + +@pytest.mark.asyncio +async def test_receive_button_event_invalid_payload(client): + mock_pub_socket = AsyncMock() + client.app.state.endpoints_pub_socket = mock_pub_socket + + # Missing context + payload = {"type": "speech"} + response = client.post("/button_pressed", json=payload) + + assert response.status_code == 422 + mock_pub_socket.send_multipart.assert_not_called() + + +@pytest.mark.asyncio +async def test_experiment_stream_direct_call(): + """ + Directly calling the endpoint function to test the streaming logic + without dealing with TestClient streaming limitations. + """ + mock_socket = AsyncMock() + # 1. recv data + # 2. recv timeout + # 3. disconnect (request.is_disconnected returns True) + mock_socket.recv_multipart.side_effect = [ + (b"topic", b"message1"), + TimeoutError(), + (b"topic", b"message2"), # Should not be reached if disconnect checks work + ] + mock_socket.close = MagicMock() + mock_socket.connect = MagicMock() + mock_socket.subscribe = MagicMock() + + mock_context = MagicMock() + mock_context.socket.return_value = mock_socket + + with patch( + "control_backend.api.v1.endpoints.user_interact.Context.instance", return_value=mock_context + ): + mock_request = AsyncMock() + # is_disconnected sequence: + # 1. False (before first recv) -> reads message1 + # 2. False (before second recv) -> triggers TimeoutError, continues + # 3. True (before third recv) -> break loop + mock_request.is_disconnected.side_effect = [False, False, True] + + response = await user_interact.experiment_stream(mock_request) + + lines = [] + # Consume the generator + async for line in response.body_iterator: + lines.append(line) + + assert "data: message1\n\n" in lines + assert len(lines) == 1 + + mock_socket.connect.assert_called() + mock_socket.subscribe.assert_called_with(b"experiment") + mock_socket.close.assert_called() diff --git a/test/unit/test_main_sockets.py b/test/unit/test_main_sockets.py new file mode 100644 index 0000000..662147a --- /dev/null +++ b/test/unit/test_main_sockets.py @@ -0,0 +1,40 @@ +from unittest.mock import MagicMock, patch + +import zmq + +from control_backend.main import setup_sockets + + +def test_setup_sockets_proxy(): + mock_context = MagicMock() + mock_pub = MagicMock() + mock_sub = MagicMock() + + mock_context.socket.side_effect = [mock_pub, mock_sub] + + with patch("zmq.asyncio.Context.instance", return_value=mock_context): + with patch("zmq.proxy") as mock_proxy: + setup_sockets() + + mock_pub.bind.assert_called() + mock_sub.bind.assert_called() + mock_proxy.assert_called_with(mock_sub, mock_pub) + + # Check cleanup + mock_pub.close.assert_called() + mock_sub.close.assert_called() + + +def test_setup_sockets_proxy_error(): + mock_context = MagicMock() + mock_pub = MagicMock() + mock_sub = MagicMock() + mock_context.socket.side_effect = [mock_pub, mock_sub] + + with patch("zmq.asyncio.Context.instance", return_value=mock_context): + with patch("zmq.proxy", side_effect=zmq.ZMQError): + with patch("control_backend.main.logger") as mock_logger: + setup_sockets() + mock_logger.warning.assert_called() + mock_pub.close.assert_called() + mock_sub.close.assert_called() From 05804c158d87561c9336cd25720d27ca6397c15a Mon Sep 17 00:00:00 2001 From: Storm Date: Fri, 16 Jan 2026 13:26:53 +0100 Subject: [PATCH 73/90] feat: fully implemented visual emotion recognition agent in pipeline ref: N25B-393 --- .../agents/bdi/bdi_core_agent.py | 3 + .../communication/ri_communication_agent.py | 10 +++ .../visual_emotion_recognition_agent.py | 62 ++++++++++++++----- .../visual_emotion_recognizer.py | 8 +-- src/control_backend/main.py | 11 +--- src/control_backend/schemas/belief_message.py | 2 + 6 files changed, 67 insertions(+), 29 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index f056e09..bb73175 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -158,6 +158,9 @@ class BDICoreAgent(BaseAgent): for belief in beliefs: if belief.replace: self._remove_all_with_name(belief.name) + elif belief.remove: + self._remove_belief(belief.name, belief.arguments) + continue self._add_belief(belief.name, belief.arguments) def _add_belief(self, name: str, args: Iterable[str] = []): diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 5c6ca77..e01764d 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -7,6 +7,9 @@ from zmq.asyncio import Context from control_backend.agents import BaseAgent from control_backend.agents.actuation.robot_gesture_agent import RobotGestureAgent +from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognition_agent import ( + VisualEmotionRecognitionAgent, +) from control_backend.core.config import settings from ..actuation.robot_speech_agent import RobotSpeechAgent @@ -201,6 +204,13 @@ class RICommunicationAgent(BaseAgent): case "audio": vad_agent = VADAgent(audio_in_address=addr, audio_in_bind=bind) await vad_agent.start() + case "video": + visual_emotion_agent = VisualEmotionRecognitionAgent( + settings.agent_settings.visual_emotion_recognition_name, + socket_address=addr, + bind=bind, + ) + await visual_emotion_agent.start() case _: self.logger.warning("Unhandled negotiation id: %s", id) diff --git a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py index 1087138..4233e20 100644 --- a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py @@ -1,23 +1,28 @@ -import asyncio +import json +import time +from collections import Counter, defaultdict + +import cv2 +import numpy as np +from pydantic_core import ValidationError import zmq import zmq.asyncio as azmq -import numpy as np -import cv2 -from collections import defaultdict, Counter -import time from control_backend.agents import BaseAgent -from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognizer import DeepFaceEmotionRecognizer +from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognizer import ( + DeepFaceEmotionRecognizer, +) from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings +from control_backend.schemas.belief_message import Belief # START FROM RI COMMUNICATION AGENT? class VisualEmotionRecognitionAgent(BaseAgent): - def __init__(self, socket_address: str, socket_bind: bool = False, timeout_ms: int = 1000): - super().__init__(settings.agent_settings.visual_emotion_recognition_name) + def __init__(self, name, socket_address: str, bind: bool = False, timeout_ms: int = 1000): + super().__init__(name) self.socket_address = socket_address - self.socket_bind = socket_bind + self.socket_bind = bind self.timeout_ms = timeout_ms async def setup(self): @@ -41,8 +46,6 @@ class VisualEmotionRecognitionAgent(BaseAgent): async def emotion_update_loop(self): """ Retrieve a video frame from the input socket. - - :return: The received video frame, or None if timeout occurs. """ window_duration = 1 # seconds next_window_time = time.time() + window_duration @@ -70,7 +73,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): if frame_image is None: # Could not decode image, skip this frame continue - + # Get the dominant emotion from each face current_emotions = self.emotion_recognizer.sorted_dominant_emotions(frame_image) # Update emotion counts for each detected face @@ -90,7 +93,6 @@ class VisualEmotionRecognitionAgent(BaseAgent): window_dominant_emotions.add(dominant_emotion) await self.update_emotions(prev_dominant_emotions, window_dominant_emotions) - prev_dominant_emotions = window_dominant_emotions face_stats.clear() next_window_time = time.time() + window_duration @@ -98,14 +100,40 @@ class VisualEmotionRecognitionAgent(BaseAgent): except zmq.Again: self.logger.warning("No video frame received within timeout.") - async def update_emotions(self, prev_emotions, emotions): + async def update_emotions(self, prev_emotions: set[str], emotions: set[str]): + """ + Compare emotions from previous window and current emotions, + send updates to BDI Core Agent. + """ # Remove emotions that are no longer present emotions_to_remove = prev_emotions - emotions + new_emotions = emotions - prev_emotions + + if not new_emotions and not emotions_to_remove: + return + + emotion_beliefs = [] + # Remove emotions that have disappeared for emotion in emotions_to_remove: self.logger.info(f"Emotion '{emotion}' has disappeared.") - + try: + emotion_beliefs.append(Belief(name="emotion", arguments=[emotion], remove=True)) + except ValidationError: + self.logger.warning("Invalid belief for emotion removal: %s", emotion) + # Add new emotions that have appeared - new_emotions = emotions - prev_emotions for emotion in new_emotions: self.logger.info(f"New emotion detected: '{emotion}'") - \ No newline at end of file + try: + emotion_beliefs.append(Belief(name="emotion", arguments=[emotion])) + except ValidationError: + self.logger.warning("Invalid belief for new emotion: %s", emotion) + + message = InternalMessage( + to=settings.agent_settings.bdi_core_name, + sender=self.name, + body=json.dumps(emotion_beliefs), + thread="beliefs", + ) + self.logger.debug("Sending emotion beliefs update: %s", emotion_beliefs) + await self.send(message) \ No newline at end of file diff --git a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py index 06e7e4d..7bfa2e5 100644 --- a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py +++ b/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py @@ -1,7 +1,8 @@ -import abc -from deepface import DeepFace +import abc + import numpy as np -from collections import Counter +from deepface import DeepFace + class VisualEmotionRecognizer(abc.ABC): @abc.abstractmethod @@ -42,7 +43,6 @@ class DeepFaceEmotionRecognizer(VisualEmotionRecognizer): analysis = [face for face in analysis if face['face_confidence'] >= 0.90] - # Return list of (dominant_emotion, face_confidence) tuples dominant_emotions = [face['dominant_emotion'] for face in analysis] return dominant_emotions diff --git a/src/control_backend/main.py b/src/control_backend/main.py index ce2b852..714515b 100644 --- a/src/control_backend/main.py +++ b/src/control_backend/main.py @@ -40,7 +40,9 @@ from control_backend.agents.communication import RICommunicationAgent from control_backend.agents.llm import LLMAgent # User Interrupt Agent -from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognition_agent import VisualEmotionRecognitionAgent +from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognition_agent import ( + VisualEmotionRecognitionAgent, +) from control_backend.agents.user_interrupt.user_interrupt_agent import UserInterruptAgent # Other backend imports @@ -148,13 +150,6 @@ async def lifespan(app: FastAPI): "name": settings.agent_settings.user_interrupt_name, }, ), - # TODO: Spawn agent from RI Communication Agent - "VisualEmotionRecognitionAgent": ( - VisualEmotionRecognitionAgent, - { - "socket_address": "tcp://localhost:5556", # TODO: move to settings - }, - ), } agents = [] diff --git a/src/control_backend/schemas/belief_message.py b/src/control_backend/schemas/belief_message.py index deb1152..07b7237 100644 --- a/src/control_backend/schemas/belief_message.py +++ b/src/control_backend/schemas/belief_message.py @@ -8,11 +8,13 @@ class Belief(BaseModel): :ivar name: The functor or name of the belief (e.g., 'user_said'). :ivar arguments: A list of string arguments for the belief. :ivar replace: If True, existing beliefs with this name should be replaced by this one. + :ivar remove: If True, this belief should be removed from the belief base. """ name: str arguments: list[str] replace: bool = False + remove: bool = False class BeliefMessage(BaseModel): From 6d03ba8a4153015cda8c8ec4ba1372233c7084af Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Fri, 16 Jan 2026 14:28:27 +0100 Subject: [PATCH 74/90] feat: added extra endpoint for norm pings also made sure that you cannot skip phase on end phase ref: N25B-400 --- .../agents/bdi/agentspeak_generator.py | 10 ++++++ .../user_interrupt/user_interrupt_agent.py | 29 ++++++++++------- .../api/v1/endpoints/user_interact.py | 31 +++++++++++++++++-- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index ed6f787..524f980 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -424,6 +424,16 @@ class AgentSpeakGenerator: ) ) + # Force phase transition fallback + self._asp.plans.append( + AstPlan( + TriggerType.ADDED_GOAL, + AstLiteral("force_transition_phase"), + [], + [AstStatement(StatementType.EMPTY, AstLiteral("true"))], + ) + ) + @singledispatchmethod def _astify(self, element: ProgramElement) -> AstExpression: raise NotImplementedError(f"Cannot convert element {element} to an AgentSpeak expression.") diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 0bde563..9ba8409 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -117,13 +117,13 @@ class UserInterruptAgent(BaseAgent): event_context, ) elif asl_cond_norm := self._cond_norm_map.get(ui_id): - await self._send_to_bdi_belief(asl_cond_norm) + await self._send_to_bdi_belief(asl_cond_norm, "cond_norm") self.logger.info( "Forwarded button press (override) with context '%s' to BDI Core.", event_context, ) elif asl_goal := self._goal_map.get(ui_id): - await self._send_to_bdi_belief(asl_goal) + await self._send_to_bdi_belief(asl_goal, "goal") self.logger.info( "Forwarded button press (override) with context '%s' to BDI Core.", event_context, @@ -141,7 +141,7 @@ class UserInterruptAgent(BaseAgent): case "override_unachieve": ui_id = str(event_context) if asl_cond_norm := self._cond_norm_map.get(ui_id): - await self._send_to_bdi_belief(asl_cond_norm, True) + await self._send_to_bdi_belief(asl_cond_norm, "cond_norm", True) self.logger.info( "Forwarded button press (override_unachieve)" "with context '%s' to BDI Core.", @@ -187,11 +187,9 @@ class UserInterruptAgent(BaseAgent): payload = {"type": "trigger_update", "id": ui_id, "achieved": True} await self._send_experiment_update(payload) self.logger.info(f"UI Update: Trigger {asl_slug} started (ID: {ui_id})") - case "trigger_end": asl_slug = msg.body ui_id = self._trigger_reverse_map.get(asl_slug) - if ui_id: payload = {"type": "trigger_update", "id": ui_id, "achieved": False} await self._send_experiment_update(payload) @@ -207,7 +205,7 @@ class UserInterruptAgent(BaseAgent): goal_name = msg.body ui_id = self._goal_reverse_map.get(goal_name) if ui_id: - payload = {"type": "goal_update", "id": ui_id, "active": True} + payload = {"type": "goal_update", "id": ui_id} await self._send_experiment_update(payload) self.logger.info(f"UI Update: Goal {goal_name} started (ID: {ui_id})") case "active_norms_update": @@ -224,15 +222,17 @@ class UserInterruptAgent(BaseAgent): :param active_slugs: A list of slugs (strings) currently active in the BDI core. """ updates = [] - for asl_slug, ui_id in self._cond_norm_reverse_map.items(): is_active = asl_slug in active_slugs - updates.append({"id": ui_id, "name": asl_slug, "active": is_active}) + updates.append({"id": ui_id, "active": is_active}) payload = {"type": "cond_norms_state_update", "norms": updates} - await self._send_experiment_update(payload, should_log=False) - # self.logger.debug(f"Broadcasted state for {len(updates)} conditional norms.") + if self.pub_socket: + topic = b"status" + body = json.dumps(payload).encode("utf-8") + await self.pub_socket.send_multipart([topic, body]) + # self.logger.info(f"UI Update: Active norms {updates}") def _create_mapping(self, program_json: str): """ @@ -325,9 +325,14 @@ class UserInterruptAgent(BaseAgent): await self.send(msg) self.logger.info(f"Directly forced {thread} in BDI: {body}") - async def _send_to_bdi_belief(self, asl_goal: str, unachieve: bool = False): + async def _send_to_bdi_belief(self, asl: str, asl_type: str, unachieve: bool = False): """Send belief to BDI Core""" - belief_name = f"achieved_{asl_goal}" + if asl_type == "goal": + belief_name = f"achieved_{asl}" + elif asl_type == "cond_norm": + belief_name = f"force_{asl}" + else: + self.logger.warning("Tried to send belief with unknown type") belief = Belief(name=belief_name, arguments=None) self.logger.debug(f"Sending belief to BDI Core: {belief_name}") # Conditional norms are unachieved by removing the belief diff --git a/src/control_backend/api/v1/endpoints/user_interact.py b/src/control_backend/api/v1/endpoints/user_interact.py index 3d3406e..eb70f35 100644 --- a/src/control_backend/api/v1/endpoints/user_interact.py +++ b/src/control_backend/api/v1/endpoints/user_interact.py @@ -52,11 +52,11 @@ async def experiment_stream(request: Request): while True: # Check if client closed the tab if await request.is_disconnected(): - logger.info("Client disconnected from experiment stream.") + logger.error("Client disconnected from experiment stream.") break try: - parts = await asyncio.wait_for(socket.recv_multipart(), timeout=1.0) + parts = await asyncio.wait_for(socket.recv_multipart(), timeout=10.0) _, message = parts yield f"data: {message.decode().strip()}\n\n" except TimeoutError: @@ -65,3 +65,30 @@ async def experiment_stream(request: Request): socket.close() return StreamingResponse(gen(), media_type="text/event-stream") + + +@router.get("/status_stream") +async def status_stream(request: Request): + context = Context.instance() + socket = context.socket(zmq.SUB) + socket.connect(settings.zmq_settings.internal_sub_address) + + socket.subscribe(b"status") + + async def gen(): + try: + while True: + if await request.is_disconnected(): + break + try: + # Shorter timeout since this is frequent + parts = await asyncio.wait_for(socket.recv_multipart(), timeout=0.5) + _, message = parts + yield f"data: {message.decode().strip()}\n\n" + except TimeoutError: + yield ": ping\n\n" # Keep the connection alive + continue + finally: + socket.close() + + return StreamingResponse(gen(), media_type="text/event-stream") From 7c10c50336ebb57739d04782bf907b00c63cd866 Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Fri, 16 Jan 2026 14:29:46 +0100 Subject: [PATCH 75/90] chore: removed resetExperiment from backened now it happens in UI ref: N25B-400 --- .../agents/user_interrupt/user_interrupt_agent.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 9ba8409..cf72ce5 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -80,7 +80,6 @@ class UserInterruptAgent(BaseAgent): - type: "next_phase", context: None, indicates to the BDI Core to - type: "pause", context: boolean indicating whether to pause - type: "reset_phase", context: None, indicates to the BDI Core to - - type: "reset_experiment", context: None, indicates to the BDI Core to """ while True: topic, body = await self.sub_socket.recv_multipart() @@ -162,7 +161,7 @@ class UserInterruptAgent(BaseAgent): else: self.logger.info("Sent resume command.") - case "next_phase" | "reset_phase" | "reset_experiment": + case "next_phase" | "reset_phase": await self._send_experiment_control_to_bdi_core(event_type) case _: self.logger.warning( @@ -359,8 +358,6 @@ class UserInterruptAgent(BaseAgent): thread = "force_next_phase" case "reset_phase": thread = "reset_current_phase" - case "reset_experiment": - thread = "reset_experiment" case _: self.logger.warning( "Received unknown experiment control type '%s' to send to BDI Core.", From a09d8b3d9a177ad2d4c211a14942ea6ceb26604a Mon Sep 17 00:00:00 2001 From: Storm Date: Fri, 16 Jan 2026 14:40:59 +0100 Subject: [PATCH 76/90] chore: small changes --- .../communication/ri_communication_agent.py | 2 +- .../visual_emotion_recognition_agent.py | 15 ++++++++------- .../visual_emotion_recognizer.py | 0 src/control_backend/main.py | 3 --- 4 files changed, 9 insertions(+), 11 deletions(-) rename src/control_backend/agents/perception/{visual_emotion_detection_agent => visual_emotion_recognition_agent}/visual_emotion_recognition_agent.py (92%) rename src/control_backend/agents/perception/{visual_emotion_detection_agent => visual_emotion_recognition_agent}/visual_emotion_recognizer.py (100%) diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index e01764d..02ebed5 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -7,7 +7,7 @@ from zmq.asyncio import Context from control_backend.agents import BaseAgent from control_backend.agents.actuation.robot_gesture_agent import RobotGestureAgent -from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognition_agent import ( +from control_backend.agents.perception.visual_emotion_recognition_agent.visual_emotion_recognition_agent import ( VisualEmotionRecognitionAgent, ) from control_backend.core.config import settings diff --git a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py similarity index 92% rename from src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py rename to src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py index 4233e20..a15462b 100644 --- a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py @@ -9,7 +9,7 @@ import zmq import zmq.asyncio as azmq from control_backend.agents import BaseAgent -from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognizer import ( +from control_backend.agents.perception.visual_emotion_recognition_agent.visual_emotion_recognizer import ( DeepFaceEmotionRecognizer, ) from control_backend.core.agent_system import InternalMessage @@ -105,11 +105,10 @@ class VisualEmotionRecognitionAgent(BaseAgent): Compare emotions from previous window and current emotions, send updates to BDI Core Agent. """ - # Remove emotions that are no longer present emotions_to_remove = prev_emotions - emotions - new_emotions = emotions - prev_emotions + emotions_to_add = emotions - prev_emotions - if not new_emotions and not emotions_to_remove: + if not emotions_to_add and not emotions_to_remove: return emotion_beliefs = [] @@ -122,18 +121,20 @@ class VisualEmotionRecognitionAgent(BaseAgent): self.logger.warning("Invalid belief for emotion removal: %s", emotion) # Add new emotions that have appeared - for emotion in new_emotions: + for emotion in emotions_to_add: self.logger.info(f"New emotion detected: '{emotion}'") try: emotion_beliefs.append(Belief(name="emotion", arguments=[emotion])) except ValidationError: self.logger.warning("Invalid belief for new emotion: %s", emotion) + beliefs_list = [b.model_dump() for b in emotion_beliefs] + payload = {"beliefs": beliefs_list} + message = InternalMessage( to=settings.agent_settings.bdi_core_name, sender=self.name, - body=json.dumps(emotion_beliefs), + body=json.dumps(payload), thread="beliefs", ) - self.logger.debug("Sending emotion beliefs update: %s", emotion_beliefs) await self.send(message) \ No newline at end of file diff --git a/src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py similarity index 100% rename from src/control_backend/agents/perception/visual_emotion_detection_agent/visual_emotion_recognizer.py rename to src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py diff --git a/src/control_backend/main.py b/src/control_backend/main.py index 714515b..3509cbc 100644 --- a/src/control_backend/main.py +++ b/src/control_backend/main.py @@ -40,9 +40,6 @@ from control_backend.agents.communication import RICommunicationAgent from control_backend.agents.llm import LLMAgent # User Interrupt Agent -from control_backend.agents.perception.visual_emotion_detection_agent.visual_emotion_recognition_agent import ( - VisualEmotionRecognitionAgent, -) from control_backend.agents.user_interrupt.user_interrupt_agent import UserInterruptAgent # Other backend imports From 0941b26703f632b2f2e60c0af3e1a8f74801e7ea Mon Sep 17 00:00:00 2001 From: Storm Date: Fri, 16 Jan 2026 15:05:13 +0100 Subject: [PATCH 77/90] refactor: updated how changes are passed to bdi_core_agent after merge ref: N25B-393 --- .../visual_emotion_recognition_agent.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py index a15462b..84691ae 100644 --- a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py @@ -111,25 +111,27 @@ class VisualEmotionRecognitionAgent(BaseAgent): if not emotions_to_add and not emotions_to_remove: return - emotion_beliefs = [] + emotion_beliefs_remove = [] # Remove emotions that have disappeared for emotion in emotions_to_remove: self.logger.info(f"Emotion '{emotion}' has disappeared.") try: - emotion_beliefs.append(Belief(name="emotion", arguments=[emotion], remove=True)) + emotion_beliefs_remove.append(Belief(name="emotion", arguments=[emotion], remove=True)) except ValidationError: self.logger.warning("Invalid belief for emotion removal: %s", emotion) + emotion_beliefs_add = [] # Add new emotions that have appeared for emotion in emotions_to_add: self.logger.info(f"New emotion detected: '{emotion}'") try: - emotion_beliefs.append(Belief(name="emotion", arguments=[emotion])) + emotion_beliefs_add.append(Belief(name="emotion", arguments=[emotion])) except ValidationError: self.logger.warning("Invalid belief for new emotion: %s", emotion) - beliefs_list = [b.model_dump() for b in emotion_beliefs] - payload = {"beliefs": beliefs_list} + beliefs_list_add = [b.model_dump() for b in emotion_beliefs_add] + beliefs_list_remove = [b.model_dump() for b in emotion_beliefs_remove] + payload = {"create": beliefs_list_add, "delete": beliefs_list_remove, "replace": []} message = InternalMessage( to=settings.agent_settings.bdi_core_name, From 8506c0d9effe74889cbe2080786f948e89194caa Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Fri, 16 Jan 2026 15:07:44 +0100 Subject: [PATCH 78/90] chore: remove belief collector and small tweaks --- .../agents/bdi/bdi_core_agent.py | 2 +- src/control_backend/core/config.py | 2 -- src/control_backend/main.py | 7 ------ .../actuation/test_robot_gesture_agent.py | 3 +-- test/unit/agents/bdi/test_bdi_core_agent.py | 10 ++++---- .../agents/bdi/test_text_belief_extractor.py | 24 +++++++++++++++++++ .../test_speech_recognizer.py | 4 +++- .../perception/vad_agent/test_vad_agent.py | 1 - test/unit/conftest.py | 1 - 9 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 0c217dc..628bb53 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -167,7 +167,7 @@ class BDICoreAgent(BaseAgent): case "force_next_phase": self._force_next_phase() case _: - self.logger.warning("Received unknow user interruption: %s", msg) + self.logger.warning("Received unknown user interruption: %s", msg) def _apply_belief_changes(self, belief_changes: BeliefMessage): """ diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index 6deb1b8..329a246 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -35,7 +35,6 @@ class AgentSettings(BaseModel): Names of the various agents in the system. These names are used for routing messages. :ivar bdi_core_name: Name of the BDI Core Agent. - :ivar bdi_belief_collector_name: Name of the Belief Collector Agent. :ivar bdi_program_manager_name: Name of the BDI Program Manager Agent. :ivar text_belief_extractor_name: Name of the Text Belief Extractor Agent. :ivar vad_name: Name of the Voice Activity Detection (VAD) Agent. @@ -50,7 +49,6 @@ class AgentSettings(BaseModel): # agent names bdi_core_name: str = "bdi_core_agent" - bdi_belief_collector_name: str = "belief_collector_agent" bdi_program_manager_name: str = "bdi_program_manager_agent" text_belief_extractor_name: str = "text_belief_extractor_agent" vad_name: str = "vad_agent" diff --git a/src/control_backend/main.py b/src/control_backend/main.py index ec93b1e..a0136bd 100644 --- a/src/control_backend/main.py +++ b/src/control_backend/main.py @@ -26,7 +26,6 @@ from zmq.asyncio import Context # BDI agents from control_backend.agents.bdi import ( - BDIBeliefCollectorAgent, BDICoreAgent, TextBeliefExtractorAgent, ) @@ -122,12 +121,6 @@ async def lifespan(app: FastAPI): "name": settings.agent_settings.bdi_core_name, }, ), - "BeliefCollectorAgent": ( - BDIBeliefCollectorAgent, - { - "name": settings.agent_settings.bdi_belief_collector_name, - }, - ), "TextBeliefExtractorAgent": ( TextBeliefExtractorAgent, { diff --git a/test/unit/agents/actuation/test_robot_gesture_agent.py b/test/unit/agents/actuation/test_robot_gesture_agent.py index 225278d..1e6fd8a 100644 --- a/test/unit/agents/actuation/test_robot_gesture_agent.py +++ b/test/unit/agents/actuation/test_robot_gesture_agent.py @@ -478,8 +478,7 @@ async def test_stop_closes_sockets(): pubsocket.close.assert_called_once() subsocket.close.assert_called_once() - # Note: repsocket is not closed in stop() method, but you might want to add it - # repsocket.close.assert_called_once() + repsocket.close.assert_called_once() @pytest.mark.asyncio diff --git a/test/unit/agents/bdi/test_bdi_core_agent.py b/test/unit/agents/bdi/test_bdi_core_agent.py index 152d901..6245d5b 100644 --- a/test/unit/agents/bdi/test_bdi_core_agent.py +++ b/test/unit/agents/bdi/test_bdi_core_agent.py @@ -45,12 +45,12 @@ async def test_setup_no_asl(mock_agentspeak_env, agent): @pytest.mark.asyncio -async def test_handle_belief_collector_message(agent, mock_settings): +async def test_handle_belief_message(agent, mock_settings): """Test that incoming beliefs are added to the BDI agent""" beliefs = [Belief(name="user_said", arguments=["Hello"])] msg = InternalMessage( to="bdi_agent", - sender=mock_settings.agent_settings.bdi_belief_collector_name, + sender=mock_settings.agent_settings.text_belief_extractor_name, body=BeliefMessage(create=beliefs).model_dump_json(), thread="beliefs", ) @@ -82,7 +82,7 @@ async def test_handle_delete_belief_message(agent, mock_settings): msg = InternalMessage( to="bdi_agent", - sender=mock_settings.agent_settings.bdi_belief_collector_name, + sender=mock_settings.agent_settings.text_belief_extractor_name, body=BeliefMessage(delete=beliefs).model_dump_json(), thread="beliefs", ) @@ -104,11 +104,11 @@ async def test_handle_delete_belief_message(agent, mock_settings): @pytest.mark.asyncio -async def test_incorrect_belief_collector_message(agent, mock_settings): +async def test_incorrect_belief_message(agent, mock_settings): """Test that incorrect message format triggers an exception.""" msg = InternalMessage( to="bdi_agent", - sender=mock_settings.agent_settings.bdi_belief_collector_name, + sender=mock_settings.agent_settings.text_belief_extractor_name, body=json.dumps({"bad_format": "bad_format"}), thread="beliefs", ) diff --git a/test/unit/agents/bdi/test_text_belief_extractor.py b/test/unit/agents/bdi/test_text_belief_extractor.py index 0d7dc00..353b718 100644 --- a/test/unit/agents/bdi/test_text_belief_extractor.py +++ b/test/unit/agents/bdi/test_text_belief_extractor.py @@ -359,6 +359,30 @@ async def test_simulated_real_turn_remove_belief(agent, llm, sample_program): assert any(b.name == "no_more_booze" for b in agent._current_beliefs.false) +@pytest.mark.asyncio +async def test_infer_goal_completions_sends_beliefs(agent, llm): + """Test that inferred goal completions are sent to the BDI core.""" + goal = BaseGoal( + id=uuid.uuid4(), name="Say Hello", description="The user said hello", can_fail=True + ) + agent.goal_inferrer.goals = {goal} + + # Mock goal inference: goal is achieved + llm.query = AsyncMock(return_value=True) + + await agent._infer_goal_completions() + + # Should send belief change to BDI core + agent.send.assert_awaited_once() + sent: InternalMessage = agent.send.call_args.args[0] + assert sent.to == settings.agent_settings.bdi_core_name + assert sent.thread == "beliefs" + + parsed = BeliefMessage.model_validate_json(sent.body) + assert len(parsed.create) == 1 + assert parsed.create[0].name == "achieved_say_hello" + + @pytest.mark.asyncio async def test_llm_failure_handling(agent, llm, sample_program): """ diff --git a/test/unit/agents/perception/transcription_agent/test_speech_recognizer.py b/test/unit/agents/perception/transcription_agent/test_speech_recognizer.py index 47443a9..518d189 100644 --- a/test/unit/agents/perception/transcription_agent/test_speech_recognizer.py +++ b/test/unit/agents/perception/transcription_agent/test_speech_recognizer.py @@ -55,4 +55,6 @@ def test_get_decode_options(): assert isinstance(options["sample_len"], int) # When disabled, it should not limit output length based on input size - assert "sample_rate" not in options + recognizer = OpenAIWhisperSpeechRecognizer(limit_output_length=False) + options = recognizer._get_decode_options(audio) + assert "sample_len" not in options diff --git a/test/unit/agents/perception/vad_agent/test_vad_agent.py b/test/unit/agents/perception/vad_agent/test_vad_agent.py index fe65545..3e6b0ad 100644 --- a/test/unit/agents/perception/vad_agent/test_vad_agent.py +++ b/test/unit/agents/perception/vad_agent/test_vad_agent.py @@ -60,7 +60,6 @@ async def test_handle_message_unknown_command(agent): await agent.handle_message(msg) - agent.logger.warning.assert_called() agent._paused.clear.assert_not_called() agent._paused.set.assert_not_called() diff --git a/test/unit/conftest.py b/test/unit/conftest.py index 6ab989e..d5f06e5 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -25,7 +25,6 @@ def mock_settings(): mock.zmq_settings.internal_sub_address = "tcp://localhost:5561" mock.zmq_settings.ri_command_address = "tcp://localhost:0000" mock.agent_settings.bdi_core_name = "bdi_core_agent" - mock.agent_settings.bdi_belief_collector_name = "belief_collector_agent" mock.agent_settings.llm_name = "llm_agent" mock.agent_settings.robot_speech_name = "robot_speech_agent" mock.agent_settings.transcription_name = "transcription_agent" From 1b0b72d63a70e4b4441ccf513e1fb8f503e68740 Mon Sep 17 00:00:00 2001 From: Storm Date: Fri, 16 Jan 2026 15:10:55 +0100 Subject: [PATCH 79/90] chore: fixed broken uv.lock file --- pyproject.toml | 1 + uv.lock | 45 +++++++++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 467f2e9..b654ebc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "silero-vad>=6.0.0", "sphinx>=7.3.7", "sphinx-rtd-theme>=3.0.2", + "tf-keras>=2.20.1", "torch>=2.8.0", "uvicorn>=0.37.0", ] diff --git a/uv.lock b/uv.lock index 1f2646f..ca11c66 100644 --- a/uv.lock +++ b/uv.lock @@ -308,7 +308,7 @@ wheels = [ [[package]] name = "deepface" -version = "0.0.96" +version = "0.0.97" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fire" }, @@ -323,14 +323,15 @@ dependencies = [ { name = "opencv-python" }, { name = "pandas" }, { name = "pillow" }, + { name = "python-dotenv" }, { name = "requests" }, { name = "retina-face" }, { name = "tensorflow" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c8/20/d2a2dd919bbb9aeacf20788613f4939016dc0db3e2ebfbeff647c5e2299e/deepface-0.0.96.tar.gz", hash = "sha256:d3e7a418a48c8230621a06200f017599fbddfee0f1d2c5c8382299c7b67e6ef6", size = 112346, upload-time = "2025-11-23T14:16:29.698Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/72/63aa7cb58462ba835b263741697a3b689241175c1015b70124778d84c2bf/deepface-0.0.97.tar.gz", hash = "sha256:962b1c1c6faabfbc48c78e0dbf9d2c260136ff86bbdeb797d264f98d1a8ae6cc", size = 130063, upload-time = "2026-01-13T18:36:26.883Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/19/282ae42de09fc0e29ecfc902453315b06a22eb273782d5527223232f9949/deepface-0.0.96-py3-none-any.whl", hash = "sha256:957a30f8dbbce2ace4cea89bd16038b2a776a8c6146cf61138beba79537adf69", size = 133122, upload-time = "2025-11-23T14:16:28.472Z" }, + { url = "https://files.pythonhosted.org/packages/83/8f/76fac5130e28b45288cdf792a2a92163027dd25b3815461c9319e9f77e8e/deepface-0.0.97-py3-none-any.whl", hash = "sha256:0e78d5ee9ad2b6a03442ac2292c9aea073c6d449769ca1b16b00ea507be78434", size = 160830, upload-time = "2026-01-13T18:36:24.958Z" }, ] [[package]] @@ -777,7 +778,7 @@ wheels = [ [[package]] name = "keras" -version = "3.13.0" +version = "3.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "absl-py" }, @@ -789,9 +790,9 @@ dependencies = [ { name = "packaging" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3f/eb/960ec65476a6f5c9223fac7e6ae09999cd75a75fb87afe130e4b1d43f0ac/keras-3.13.0.tar.gz", hash = "sha256:ec51ad2ffcef086d0e3077ac461fa9e3bc54f91d94b49b7c9a84c9af7f54cf5e", size = 1153648, upload-time = "2025-12-17T23:49:25.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0c/79/721253a7b1618487a901f08c913af7cf4f60caab764ef06bfc9702280b50/keras-3.13.1.tar.gz", hash = "sha256:670c726dfc9c357fe7ae5ef1c15d8f61ee7fbb40ae9a091a458ec6444a772480", size = 1154466, upload-time = "2026-01-14T18:58:26.809Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/d2/c6734cbf15288d75722ed3eb9d8ebf9204e48379c08160fd40fcd58a0c8b/keras-3.13.0-py3-none-any.whl", hash = "sha256:096793e2be6230816f3f7e030370e66c0f4a89707c59bf2d8fad3ca33869bd1c", size = 1512339, upload-time = "2025-12-17T23:49:24.46Z" }, + { url = "https://files.pythonhosted.org/packages/d8/e5/8b40bada1f33f25deca7bad0e8c7ca6752f2b09e8018e2fc4693858dd662/keras-3.13.1-py3-none-any.whl", hash = "sha256:6593be2e9d057921cd0413b4552c05fe1df73ea4bb03a4f3ec94532077b3e508", size = 1512409, upload-time = "2026-01-14T18:58:24.769Z" }, ] [[package]] @@ -1521,6 +1522,7 @@ dependencies = [ { name = "silero-vad" }, { name = "sphinx" }, { name = "sphinx-rtd-theme" }, + { name = "tf-keras" }, { name = "torch" }, { name = "uvicorn" }, ] @@ -1572,6 +1574,7 @@ requires-dist = [ { name = "silero-vad", specifier = ">=6.0.0" }, { name = "sphinx", specifier = ">=7.3.7" }, { name = "sphinx-rtd-theme", specifier = ">=3.0.2" }, + { name = "tf-keras", specifier = ">=2.20.1" }, { name = "torch", specifier = ">=2.8.0" }, { name = "uvicorn", specifier = ">=0.37.0" }, ] @@ -1942,12 +1945,6 @@ wheels = [ ] [[package]] -name = "pytz" -version = "2025.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, name = "python-slugify" version = "8.0.4" source = { registry = "https://pypi.org/simple" } @@ -1959,6 +1956,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", size = 10051, upload-time = "2024-02-08T18:32:43.911Z" }, ] +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -2592,6 +2598,9 @@ source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, +] + +[[package]] name = "text-unidecode" version = "1.3" source = { registry = "https://pypi.org/simple" } @@ -2600,6 +2609,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, ] +[[package]] +name = "tf-keras" +version = "2.20.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tensorflow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/38/6060f6c7472439bb3890b9094d69d31d9f8d5da123b16c738773e70fff91/tf_keras-2.20.1.tar.gz", hash = "sha256:884be5938fb0b2b53b1583c1ae2b660ef87215377c29b5b6a77fd221b472aeaf", size = 1254487, upload-time = "2025-09-04T21:23:41.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/6b/d9a8202bfe5c9e3b078cf550bafab962aa9d6b1a1f1180f0065399d4c9b2/tf_keras-2.20.1-py3-none-any.whl", hash = "sha256:3f0e0a34d9a4c8758f24fdc1053e6e335f16ab5534c7d34f1899b8924779760c", size = 1694335, upload-time = "2025-09-04T21:23:40.153Z" }, +] + [[package]] name = "tiktoken" version = "0.12.0" From 7f7e0c542ee5bb61bbff2ed94662c669ff75e0ce Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Fri, 16 Jan 2026 15:35:41 +0100 Subject: [PATCH 80/90] docs: add missing docs ref: N25B-115 --- src/control_backend/agents/__init__.py | 4 + .../agents/actuation/__init__.py | 4 + src/control_backend/agents/bdi/__init__.py | 5 ++ .../agents/bdi/agentspeak_ast.py | 36 ++++++++- .../agents/bdi/agentspeak_generator.py | 14 ++++ .../agents/bdi/text_belief_extractor_agent.py | 12 ++- .../agents/communication/__init__.py | 4 + src/control_backend/agents/llm/__init__.py | 4 + .../agents/perception/__init__.py | 5 ++ .../transcription_agent.py | 2 +- .../user_interrupt/user_interrupt_agent.py | 45 ++++++----- src/control_backend/api/v1/endpoints/sse.py | 12 --- src/control_backend/core/agent_system.py | 12 +++ src/control_backend/schemas/belief_list.py | 6 ++ src/control_backend/schemas/chat_history.py | 13 ++++ src/control_backend/schemas/events.py | 8 ++ src/control_backend/schemas/program.py | 78 ++++++++++--------- 17 files changed, 191 insertions(+), 73 deletions(-) delete mode 100644 src/control_backend/api/v1/endpoints/sse.py diff --git a/src/control_backend/agents/__init__.py b/src/control_backend/agents/__init__.py index 1618d55..85f4aad 100644 --- a/src/control_backend/agents/__init__.py +++ b/src/control_backend/agents/__init__.py @@ -1 +1,5 @@ +""" +This package contains all agent implementations for the PepperPlus Control Backend. +""" + from .base import BaseAgent as BaseAgent diff --git a/src/control_backend/agents/actuation/__init__.py b/src/control_backend/agents/actuation/__init__.py index 8ff7e7f..9a8d81b 100644 --- a/src/control_backend/agents/actuation/__init__.py +++ b/src/control_backend/agents/actuation/__init__.py @@ -1,2 +1,6 @@ +""" +Agents responsible for controlling the robot's physical actions, such as speech and gestures. +""" + from .robot_gesture_agent import RobotGestureAgent as RobotGestureAgent from .robot_speech_agent import RobotSpeechAgent as RobotSpeechAgent diff --git a/src/control_backend/agents/bdi/__init__.py b/src/control_backend/agents/bdi/__init__.py index d6f5124..2f7d976 100644 --- a/src/control_backend/agents/bdi/__init__.py +++ b/src/control_backend/agents/bdi/__init__.py @@ -1,3 +1,8 @@ +""" +Agents and utilities for the BDI (Belief-Desire-Intention) reasoning system, +implementing AgentSpeak(L) logic. +""" + from control_backend.agents.bdi.bdi_core_agent import BDICoreAgent as BDICoreAgent from .text_belief_extractor_agent import ( diff --git a/src/control_backend/agents/bdi/agentspeak_ast.py b/src/control_backend/agents/bdi/agentspeak_ast.py index 68be531..19f48e2 100644 --- a/src/control_backend/agents/bdi/agentspeak_ast.py +++ b/src/control_backend/agents/bdi/agentspeak_ast.py @@ -80,7 +80,7 @@ class AstTerm(AstExpression, ABC): @dataclass(eq=False) class AstAtom(AstTerm): """ - Grounded expression in all lowercase. + Represents a grounded atom in AgentSpeak (e.g., lowercase constants). """ value: str @@ -92,7 +92,7 @@ class AstAtom(AstTerm): @dataclass(eq=False) class AstVar(AstTerm): """ - Ungrounded variable expression. First letter capitalized. + Represents an ungrounded variable in AgentSpeak (e.g., capitalized names). """ name: str @@ -103,6 +103,10 @@ class AstVar(AstTerm): @dataclass(eq=False) class AstNumber(AstTerm): + """ + Represents a numeric constant in AgentSpeak. + """ + value: int | float def _to_agentspeak(self) -> str: @@ -111,6 +115,10 @@ class AstNumber(AstTerm): @dataclass(eq=False) class AstString(AstTerm): + """ + Represents a string literal in AgentSpeak. + """ + value: str def _to_agentspeak(self) -> str: @@ -119,6 +127,10 @@ class AstString(AstTerm): @dataclass(eq=False) class AstLiteral(AstTerm): + """ + Represents a literal (functor and terms) in AgentSpeak. + """ + functor: str terms: list[AstTerm] = field(default_factory=list) @@ -142,6 +154,10 @@ class BinaryOperatorType(StrEnum): @dataclass class AstBinaryOp(AstExpression): + """ + Represents a binary logical or relational operation in AgentSpeak. + """ + left: AstExpression operator: BinaryOperatorType right: AstExpression @@ -167,6 +183,10 @@ class AstBinaryOp(AstExpression): @dataclass class AstLogicalExpression(AstExpression): + """ + Represents a logical expression, potentially negated, in AgentSpeak. + """ + expression: AstExpression negated: bool = False @@ -208,6 +228,10 @@ class AstStatement(AstNode): @dataclass class AstRule(AstNode): + """ + Represents an inference rule in AgentSpeak. If there is no condition, it always holds. + """ + result: AstExpression condition: AstExpression | None = None @@ -231,6 +255,10 @@ class TriggerType(StrEnum): @dataclass class AstPlan(AstNode): + """ + Represents a plan in AgentSpeak, consisting of a trigger, context, and body. + """ + type: TriggerType trigger_literal: AstExpression context: list[AstExpression] @@ -260,6 +288,10 @@ class AstPlan(AstNode): @dataclass class AstProgram(AstNode): + """ + Represents a full AgentSpeak program, consisting of rules and plans. + """ + rules: list[AstRule] = field(default_factory=list) plans: list[AstPlan] = field(default_factory=list) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 524f980..2fe12e3 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -40,9 +40,23 @@ from control_backend.schemas.program import ( class AgentSpeakGenerator: + """ + Generator class that translates a high-level :class:`~control_backend.schemas.program.Program` + into AgentSpeak(L) source code. + + It handles the conversion of phases, norms, goals, and triggers into AgentSpeak rules and plans, + ensuring the robot follows the defined behavioral logic. + """ + _asp: AstProgram def generate(self, program: Program) -> str: + """ + Translates a Program object into an AgentSpeak source string. + + :param program: The behavior program to translate. + :return: The generated AgentSpeak code as a string. + """ self._asp = AstProgram() if program.phases: diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index b5fd266..362dfbf 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -18,6 +18,12 @@ type JSONLike = None | bool | int | float | str | list["JSONLike"] | dict[str, " class BeliefState(BaseModel): + """ + Represents the state of inferred semantic beliefs. + + Maintains sets of beliefs that are currently considered true or false. + """ + true: set[InternalBelief] = set() false: set[InternalBelief] = set() @@ -338,7 +344,7 @@ class TextBeliefExtractorAgent(BaseAgent): class SemanticBeliefInferrer: """ - Class that handles only prompting an LLM for semantic beliefs. + Infers semantic beliefs from conversation history using an LLM. """ def __init__( @@ -464,6 +470,10 @@ Respond with a JSON similar to the following, but with the property names as giv class GoalAchievementInferrer(SemanticBeliefInferrer): + """ + Infers whether specific conversational goals have been achieved using an LLM. + """ + def __init__(self, llm: TextBeliefExtractorAgent.LLM): super().__init__(llm) self.goals: set[BaseGoal] = set() diff --git a/src/control_backend/agents/communication/__init__.py b/src/control_backend/agents/communication/__init__.py index 2aa1535..3dde6cf 100644 --- a/src/control_backend/agents/communication/__init__.py +++ b/src/control_backend/agents/communication/__init__.py @@ -1 +1,5 @@ +""" +Agents responsible for external communication and service discovery. +""" + from .ri_communication_agent import RICommunicationAgent as RICommunicationAgent diff --git a/src/control_backend/agents/llm/__init__.py b/src/control_backend/agents/llm/__init__.py index e12ff29..519812f 100644 --- a/src/control_backend/agents/llm/__init__.py +++ b/src/control_backend/agents/llm/__init__.py @@ -1 +1,5 @@ +""" +Agents that interface with Large Language Models for natural language processing and generation. +""" + from .llm_agent import LLMAgent as LLMAgent diff --git a/src/control_backend/agents/perception/__init__.py b/src/control_backend/agents/perception/__init__.py index e18361a..5a46671 100644 --- a/src/control_backend/agents/perception/__init__.py +++ b/src/control_backend/agents/perception/__init__.py @@ -1,3 +1,8 @@ +""" +Agents responsible for processing sensory input, such as audio transcription and voice activity +detection. +""" + from .transcription_agent.transcription_agent import ( TranscriptionAgent as TranscriptionAgent, ) diff --git a/src/control_backend/agents/perception/transcription_agent/transcription_agent.py b/src/control_backend/agents/perception/transcription_agent/transcription_agent.py index 765d7ac..795623d 100644 --- a/src/control_backend/agents/perception/transcription_agent/transcription_agent.py +++ b/src/control_backend/agents/perception/transcription_agent/transcription_agent.py @@ -74,7 +74,7 @@ class TranscriptionAgent(BaseAgent): def _connect_audio_in_socket(self): """ - Helper to connect the ZMQ SUB socket for audio input. + Connects the ZMQ SUB socket for receiving audio data. """ self.audio_in_socket = azmq.Context.instance().socket(zmq.SUB) self.audio_in_socket.setsockopt_string(zmq.SUBSCRIBE, "") diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index cf72ce5..6a4c9b0 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -50,10 +50,8 @@ class UserInterruptAgent(BaseAgent): async def setup(self): """ - Initialize the agent. - - Connects the internal ZMQ SUB socket and subscribes to the 'button_pressed' topic. - Starts the background behavior to receive the user interrupts. + Initialize the agent by setting up ZMQ sockets for receiving button events and + publishing updates. """ context = Context.instance() @@ -68,18 +66,15 @@ class UserInterruptAgent(BaseAgent): async def _receive_button_event(self): """ - The behaviour of the UserInterruptAgent. - Continuous loop that receives button_pressed events from the button_pressed HTTP endpoint. - These events contain a type and a context. + Main loop to receive and process button press events from the UI. - These are the different types and contexts: - - type: "speech", context: string that the robot has to say. - - type: "gesture", context: single gesture name that the robot has to perform. - - type: "override", context: id that belongs to the goal/trigger/conditional norm. - - type: "override_unachieve", context: id that belongs to the conditional norm to unachieve. - - type: "next_phase", context: None, indicates to the BDI Core to - - type: "pause", context: boolean indicating whether to pause - - type: "reset_phase", context: None, indicates to the BDI Core to + Handles different event types: + - `speech`: Triggers immediate robot speech. + - `gesture`: Triggers an immediate robot gesture. + - `override`: Forces a belief, trigger, or goal completion in the BDI core. + - `override_unachieve`: Removes a belief from the BDI core. + - `pause`: Toggles the system's pause state. + - `next_phase` / `reset_phase`: Controls experiment flow. """ while True: topic, body = await self.sub_socket.recv_multipart() @@ -172,7 +167,10 @@ class UserInterruptAgent(BaseAgent): async def handle_message(self, msg: InternalMessage): """ - Handle commands received from other internal Python agents. + Handles internal messages from other agents, such as program updates or trigger + notifications. + + :param msg: The incoming :class:`~control_backend.core.agent_system.InternalMessage`. """ match msg.thread: case "new_program": @@ -217,8 +215,9 @@ class UserInterruptAgent(BaseAgent): async def _broadcast_cond_norms(self, active_slugs: list[str]): """ - Sends the current state of all conditional norms to the UI. - :param active_slugs: A list of slugs (strings) currently active in the BDI core. + Broadcasts the current activation state of all conditional norms to the UI. + + :param active_slugs: A list of sluggified norm names currently active in the BDI core. """ updates = [] for asl_slug, ui_id in self._cond_norm_reverse_map.items(): @@ -235,7 +234,9 @@ class UserInterruptAgent(BaseAgent): def _create_mapping(self, program_json: str): """ - Create mappings between UI IDs and ASL slugs for triggers, goals, and conditional norms + Creates a bidirectional mapping between UI identifiers and AgentSpeak slugs. + + :param program_json: The JSON representation of the behavioral program. """ try: program = Program.model_validate_json(program_json) @@ -277,8 +278,10 @@ class UserInterruptAgent(BaseAgent): async def _send_experiment_update(self, data, should_log: bool = True): """ - Sends an update to the 'experiment' topic. - The SSE endpoint will pick this up and push it to the UI. + Publishes an experiment state update to the internal ZMQ bus for the UI. + + :param data: The update payload. + :param should_log: Whether to log the update. """ if self.pub_socket: topic = b"experiment" diff --git a/src/control_backend/api/v1/endpoints/sse.py b/src/control_backend/api/v1/endpoints/sse.py deleted file mode 100644 index c660aa5..0000000 --- a/src/control_backend/api/v1/endpoints/sse.py +++ /dev/null @@ -1,12 +0,0 @@ -from fastapi import APIRouter, Request - -router = APIRouter() - - -# TODO: implement -@router.get("/sse") -async def sse(request: Request): - """ - Placeholder for future Server-Sent Events endpoint. - """ - pass diff --git a/src/control_backend/core/agent_system.py b/src/control_backend/core/agent_system.py index e3c8dc4..267f072 100644 --- a/src/control_backend/core/agent_system.py +++ b/src/control_backend/core/agent_system.py @@ -22,10 +22,22 @@ class AgentDirectory: @staticmethod def register(name: str, agent: "BaseAgent"): + """ + Registers an agent instance with a unique name. + + :param name: The name of the agent. + :param agent: The :class:`BaseAgent` instance. + """ _agent_directory[name] = agent @staticmethod def get(name: str) -> "BaseAgent | None": + """ + Retrieves a registered agent instance by name. + + :param name: The name of the agent to retrieve. + :return: The :class:`BaseAgent` instance, or None if not found. + """ return _agent_directory.get(name) diff --git a/src/control_backend/schemas/belief_list.py b/src/control_backend/schemas/belief_list.py index f3d6818..841a4ed 100644 --- a/src/control_backend/schemas/belief_list.py +++ b/src/control_backend/schemas/belief_list.py @@ -16,4 +16,10 @@ class BeliefList(BaseModel): class GoalList(BaseModel): + """ + Represents a list of goals, used for communicating multiple goals between agents. + + :ivar goals: The list of goals. + """ + goals: list[BaseGoal] diff --git a/src/control_backend/schemas/chat_history.py b/src/control_backend/schemas/chat_history.py index 52fc224..8fd1e72 100644 --- a/src/control_backend/schemas/chat_history.py +++ b/src/control_backend/schemas/chat_history.py @@ -2,9 +2,22 @@ from pydantic import BaseModel class ChatMessage(BaseModel): + """ + Represents a single message in a conversation. + + :ivar role: The role of the speaker (e.g., 'user', 'assistant'). + :ivar content: The text content of the message. + """ + role: str content: str class ChatHistory(BaseModel): + """ + Represents a sequence of chat messages, forming a conversation history. + + :ivar messages: An ordered list of :class:`ChatMessage` objects. + """ + messages: list[ChatMessage] diff --git a/src/control_backend/schemas/events.py b/src/control_backend/schemas/events.py index 46967f7..a01b668 100644 --- a/src/control_backend/schemas/events.py +++ b/src/control_backend/schemas/events.py @@ -2,5 +2,13 @@ from pydantic import BaseModel class ButtonPressedEvent(BaseModel): + """ + Represents a button press event from the UI. + + :ivar type: The type of event (e.g., 'speech', 'gesture', 'override'). + :ivar context: Additional data associated with the event (e.g., speech text, gesture name, + or ID). + """ + type: str context: str diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index d04abbb..283e17d 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -20,6 +20,10 @@ class ProgramElement(BaseModel): class LogicalOperator(Enum): + """ + Logical operators for combining beliefs. + """ + AND = "AND" OR = "OR" @@ -30,9 +34,9 @@ type BasicBelief = KeywordBelief | SemanticBelief class KeywordBelief(ProgramElement): """ - Represents a belief that is set when the user spoken text contains a certain keyword. + Represents a belief that is activated when a specific keyword is detected in the user's speech. - :ivar keyword: The keyword on which this belief gets set. + :ivar keyword: The string to look for in the transcription. """ name: str = "" @@ -41,9 +45,11 @@ class KeywordBelief(ProgramElement): class SemanticBelief(ProgramElement): """ - Represents a belief that is set by semantic LLM validation. + Represents a belief whose truth value is determined by an LLM analyzing the conversation + context. - :ivar description: Description of how to form the belief, used by the LLM. + :ivar description: A natural language description of what this belief represents, + used as a prompt for the LLM. """ description: str @@ -51,13 +57,11 @@ class SemanticBelief(ProgramElement): class InferredBelief(ProgramElement): """ - Represents a belief that gets formed by combining two beliefs with a logical AND or OR. + Represents a belief derived from other beliefs using logical operators. - These beliefs can also be :class:`InferredBelief`, leading to arbitrarily deep nesting. - - :ivar operator: The logical operator to apply. - :ivar left: The left part of the logical expression. - :ivar right: The right part of the logical expression. + :ivar operator: The :class:`LogicalOperator` (AND/OR) to apply. + :ivar left: The left operand (another belief). + :ivar right: The right operand (another belief). """ name: str = "" @@ -67,6 +71,13 @@ class InferredBelief(ProgramElement): class Norm(ProgramElement): + """ + Base class for behavioral norms that guide the robot's interactions. + + :ivar norm: The textual description of the norm. + :ivar critical: Whether this norm is considered critical and should be strictly enforced. + """ + name: str = "" norm: str critical: bool = False @@ -74,10 +85,7 @@ class Norm(ProgramElement): class BasicNorm(Norm): """ - Represents a behavioral norm. - - :ivar norm: The actual norm text describing the behavior. - :ivar critical: When true, this norm should absolutely not be violated (checked separately). + A simple behavioral norm that is always considered for activation when its phase is active. """ pass @@ -85,9 +93,9 @@ class BasicNorm(Norm): class ConditionalNorm(Norm): """ - Represents a norm that is only active when a condition is met (i.e., a certain belief holds). + A behavioral norm that is only active when a specific condition (belief) is met. - :ivar condition: When to activate this norm. + :ivar condition: The :class:`Belief` that must hold for this norm to be active. """ condition: Belief @@ -140,9 +148,9 @@ type Action = SpeechAction | GestureAction | LLMAction class SpeechAction(ProgramElement): """ - Represents the action of the robot speaking a literal text. + An action where the robot speaks a predefined literal text. - :ivar text: The text to speak. + :ivar text: The text content to be spoken. """ name: str = "" @@ -151,11 +159,10 @@ class SpeechAction(ProgramElement): class Gesture(BaseModel): """ - Represents a gesture to be performed. Can be either a single gesture, - or a random gesture from a category (tag). + Defines a physical gesture for the robot to perform. - :ivar type: The type of the gesture, "tag" or "single". - :ivar name: The name of the single gesture or tag. + :ivar type: Whether to use a specific "single" gesture or a random one from a "tag" category. + :ivar name: The identifier for the gesture or tag. """ type: Literal["tag", "single"] @@ -164,9 +171,9 @@ class Gesture(BaseModel): class GestureAction(ProgramElement): """ - Represents the action of the robot performing a gesture. + An action where the robot performs a physical gesture. - :ivar gesture: The gesture to perform. + :ivar gesture: The :class:`Gesture` definition. """ name: str = "" @@ -175,10 +182,9 @@ class GestureAction(ProgramElement): class LLMAction(ProgramElement): """ - Represents the action of letting an LLM generate a reply based on its chat history - and an additional goal added in the prompt. + An action that triggers an LLM-generated conversational response. - :ivar goal: The extra (temporary) goal to add to the LLM. + :ivar goal: A temporary conversational goal to guide the LLM's response generation. """ name: str = "" @@ -187,10 +193,10 @@ class LLMAction(ProgramElement): class Trigger(ProgramElement): """ - Represents a belief-based trigger. When a belief is set, the corresponding plan is executed. + Defines a reactive behavior: when the condition (belief) is met, the plan is executed. - :ivar condition: When to activate the trigger. - :ivar plan: The plan to execute. + :ivar condition: The :class:`Belief` that triggers this behavior. + :ivar plan: The :class:`Plan` to execute upon activation. """ condition: Belief @@ -199,11 +205,11 @@ class Trigger(ProgramElement): class Phase(ProgramElement): """ - A distinct phase within a program, containing norms, goals, and triggers. + A logical stage in the interaction program, grouping norms, goals, and triggers. - :ivar norms: List of norms active in this phase. - :ivar goals: List of goals to pursue in this phase. - :ivar triggers: List of triggers that define transitions out of this phase. + :ivar norms: List of norms active during this phase. + :ivar goals: List of goals the robot pursues in this phase. + :ivar triggers: List of reactive behaviors defined for this phase. """ name: str = "" @@ -214,9 +220,9 @@ class Phase(ProgramElement): class Program(BaseModel): """ - Represents a complete interaction program, consisting of a sequence or set of phases. + The top-level container for a complete robot behavior definition. - :ivar phases: The list of phases that make up the program. + :ivar phases: An ordered list of :class:`Phase` objects defining the interaction flow. """ phases: list[Phase] From db64eaeb0b03e683e23950a13d8b4a271f17136b Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Fri, 16 Jan 2026 16:18:36 +0100 Subject: [PATCH 81/90] fix: failing tests and warnings ref: N25B-449 --- pyproject.toml | 1 + .../user_interrupt/user_interrupt_agent.py | 4 +++- src/control_backend/api/v1/router.py | 4 +--- .../perception/vad_agent/test_vad_agent.py | 7 +++--- ...st_vad_agent.py => test_vad_agent_unit.py} | 0 .../user_interrupt/test_user_interrupt.py | 4 ++-- test/unit/api/v1/endpoints/test_router.py | 1 - .../api/v1/endpoints/test_sse_endpoint.py | 24 ------------------- uv.lock | 4 +++- 9 files changed, 14 insertions(+), 35 deletions(-) rename test/unit/agents/perception/vad_agent/{test_vad_agent.py => test_vad_agent_unit.py} (100%) delete mode 100644 test/unit/api/v1/endpoints/test_sse_endpoint.py diff --git a/pyproject.toml b/pyproject.toml index cdc2ce3..5de7daa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ test = [ "pytest-asyncio>=1.2.0", "pytest-cov>=7.0.0", "pytest-mock>=3.15.1", + "python-slugify>=8.0.4", "pyyaml>=6.0.3", "pyzmq>=27.1.0", "soundfile>=0.13.1", diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index 6a4c9b0..a42861a 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -202,7 +202,7 @@ class UserInterruptAgent(BaseAgent): goal_name = msg.body ui_id = self._goal_reverse_map.get(goal_name) if ui_id: - payload = {"type": "goal_update", "id": ui_id} + payload = {"type": "goal_update", "id": ui_id, "active": True} await self._send_experiment_update(payload) self.logger.info(f"UI Update: Goal {goal_name} started (ID: {ui_id})") case "active_norms_update": @@ -361,6 +361,8 @@ class UserInterruptAgent(BaseAgent): thread = "force_next_phase" case "reset_phase": thread = "reset_current_phase" + case "reset_experiment": + thread = "reset_experiment" case _: self.logger.warning( "Received unknown experiment control type '%s' to send to BDI Core.", diff --git a/src/control_backend/api/v1/router.py b/src/control_backend/api/v1/router.py index c130ad3..b46df5f 100644 --- a/src/control_backend/api/v1/router.py +++ b/src/control_backend/api/v1/router.py @@ -1,13 +1,11 @@ from fastapi.routing import APIRouter -from control_backend.api.v1.endpoints import logs, message, program, robot, sse, user_interact +from control_backend.api.v1.endpoints import logs, message, program, robot, user_interact api_router = APIRouter() api_router.include_router(message.router, tags=["Messages"]) -api_router.include_router(sse.router, tags=["SSE"]) - api_router.include_router(robot.router, prefix="/robot", tags=["Pings", "Commands"]) api_router.include_router(logs.router, tags=["Logs"]) diff --git a/test/integration/agents/perception/vad_agent/test_vad_agent.py b/test/integration/agents/perception/vad_agent/test_vad_agent.py index 668d1ce..3cde755 100644 --- a/test/integration/agents/perception/vad_agent/test_vad_agent.py +++ b/test/integration/agents/perception/vad_agent/test_vad_agent.py @@ -40,7 +40,7 @@ async def test_normal_setup(per_transcription_agent): per_vad_agent = VADAgent("tcp://localhost:12345", False) per_vad_agent._streaming_loop = AsyncMock() - async def swallow_background_task(coro): + def swallow_background_task(coro): coro.close() per_vad_agent.add_behavior = swallow_background_task @@ -106,7 +106,7 @@ async def test_out_socket_creation_failure(zmq_context): per_vad_agent._streaming_loop = AsyncMock() per_vad_agent._connect_audio_out_socket = MagicMock(return_value=None) - async def swallow_background_task(coro): + def swallow_background_task(coro): coro.close() per_vad_agent.add_behavior = swallow_background_task @@ -126,7 +126,7 @@ async def test_stop(zmq_context, per_transcription_agent): per_vad_agent._reset_stream = AsyncMock() per_vad_agent._streaming_loop = AsyncMock() - async def swallow_background_task(coro): + def swallow_background_task(coro): coro.close() per_vad_agent.add_behavior = swallow_background_task @@ -150,6 +150,7 @@ async def test_application_startup_complete(zmq_context): vad_agent._running = True vad_agent._reset_stream = AsyncMock() vad_agent.program_sub_socket = AsyncMock() + vad_agent.program_sub_socket.close = MagicMock() vad_agent.program_sub_socket.recv_multipart.side_effect = [ (PROGRAM_STATUS, ProgramStatus.RUNNING.value), ] diff --git a/test/unit/agents/perception/vad_agent/test_vad_agent.py b/test/unit/agents/perception/vad_agent/test_vad_agent_unit.py similarity index 100% rename from test/unit/agents/perception/vad_agent/test_vad_agent.py rename to test/unit/agents/perception/vad_agent/test_vad_agent_unit.py diff --git a/test/unit/agents/user_interrupt/test_user_interrupt.py b/test/unit/agents/user_interrupt/test_user_interrupt.py index 7c38a05..7a71891 100644 --- a/test/unit/agents/user_interrupt/test_user_interrupt.py +++ b/test/unit/agents/user_interrupt/test_user_interrupt.py @@ -63,7 +63,7 @@ async def test_send_to_bdi_belief(agent): """Verify belief update format.""" context_str = "some_goal" - await agent._send_to_bdi_belief(context_str) + await agent._send_to_bdi_belief(context_str, "goal") assert agent.send.await_count == 1 sent_msg = agent.send.call_args.args[0] @@ -115,7 +115,7 @@ async def test_receive_loop_routing_success(agent): agent._send_to_gesture_agent.assert_awaited_once_with("Hello Gesture") # Override (since we mapped it to a goal) - agent._send_to_bdi_belief.assert_awaited_once_with("some_goal_slug") + agent._send_to_bdi_belief.assert_awaited_once_with("some_goal_slug", "goal") assert agent._send_to_speech_agent.await_count == 1 assert agent._send_to_gesture_agent.await_count == 1 diff --git a/test/unit/api/v1/endpoints/test_router.py b/test/unit/api/v1/endpoints/test_router.py index 7303d9c..dd93d8d 100644 --- a/test/unit/api/v1/endpoints/test_router.py +++ b/test/unit/api/v1/endpoints/test_router.py @@ -11,6 +11,5 @@ def test_router_includes_expected_paths(): # Ensure at least one route under each prefix exists assert any(p.startswith("/robot") for p in paths) assert any(p.startswith("/message") for p in paths) - assert any(p.startswith("/sse") for p in paths) assert any(p.startswith("/logs") for p in paths) assert any(p.startswith("/program") for p in paths) diff --git a/test/unit/api/v1/endpoints/test_sse_endpoint.py b/test/unit/api/v1/endpoints/test_sse_endpoint.py deleted file mode 100644 index 75a4555..0000000 --- a/test/unit/api/v1/endpoints/test_sse_endpoint.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from control_backend.api.v1.endpoints import sse - - -@pytest.fixture -def app(): - app = FastAPI() - app.include_router(sse.router) - return app - - -@pytest.fixture -def client(app): - return TestClient(app) - - -def test_sse_route_exists(client): - """Minimal smoke test to ensure /sse route exists and responds.""" - response = client.get("/sse") - # Since implementation is not done, we only assert it doesn't crash - assert response.status_code == 200 diff --git a/uv.lock b/uv.lock index ce46ceb..ea39c17 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.13" resolution-markers = [ "python_full_version >= '3.14'", @@ -1030,6 +1030,7 @@ test = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-mock" }, + { name = "python-slugify" }, { name = "pyyaml" }, { name = "pyzmq" }, { name = "soundfile" }, @@ -1080,6 +1081,7 @@ test = [ { name = "pytest-asyncio", specifier = ">=1.2.0" }, { name = "pytest-cov", specifier = ">=7.0.0" }, { name = "pytest-mock", specifier = ">=3.15.1" }, + { name = "python-slugify", specifier = ">=8.0.4" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "pyzmq", specifier = ">=27.1.0" }, { name = "soundfile", specifier = ">=0.13.1" }, From 302c50934ebde7d2999a99c27efc25d275e37a02 Mon Sep 17 00:00:00 2001 From: Storm Date: Mon, 19 Jan 2026 12:10:58 +0100 Subject: [PATCH 82/90] feat: implemented emotion recognition functionality in AgentSpeak ref: N25B-393 --- .../agents/bdi/agentspeak_generator.py | 5 +++++ .../visual_emotion_recognition_agent.py | 8 ++++---- .../visual_emotion_recognizer.py | 2 ++ src/control_backend/schemas/program.py | 19 +++++++++++++++++-- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/control_backend/agents/bdi/agentspeak_generator.py b/src/control_backend/agents/bdi/agentspeak_generator.py index 2fe12e3..93c41af 100644 --- a/src/control_backend/agents/bdi/agentspeak_generator.py +++ b/src/control_backend/agents/bdi/agentspeak_generator.py @@ -22,6 +22,7 @@ from control_backend.schemas.program import ( BaseGoal, BasicNorm, ConditionalNorm, + EmotionBelief, GestureAction, Goal, InferredBelief, @@ -459,6 +460,10 @@ class AgentSpeakGenerator: @_astify.register def _(self, sb: SemanticBelief) -> AstExpression: return AstLiteral(self.slugify(sb)) + + @_astify.register + def _(self, eb: EmotionBelief) -> AstExpression: + return AstLiteral("emotion_detected", [AstAtom(eb.emotion)]) @_astify.register def _(self, ib: InferredBelief) -> AstExpression: diff --git a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py index 84691ae..6b11015 100644 --- a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py @@ -47,7 +47,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): """ Retrieve a video frame from the input socket. """ - window_duration = 1 # seconds + window_duration = 5 # seconds next_window_time = time.time() + window_duration # To detect false positives @@ -82,7 +82,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): # If window duration has passed, process the collected stats if time.time() >= next_window_time: - + print(face_stats) window_dominant_emotions = set() # Determine dominant emotion for each face in the window for _, counter in face_stats.items(): @@ -116,7 +116,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): for emotion in emotions_to_remove: self.logger.info(f"Emotion '{emotion}' has disappeared.") try: - emotion_beliefs_remove.append(Belief(name="emotion", arguments=[emotion], remove=True)) + emotion_beliefs_remove.append(Belief(name="emotion_detected", arguments=[emotion], remove=True)) except ValidationError: self.logger.warning("Invalid belief for emotion removal: %s", emotion) @@ -125,7 +125,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): for emotion in emotions_to_add: self.logger.info(f"New emotion detected: '{emotion}'") try: - emotion_beliefs_add.append(Belief(name="emotion", arguments=[emotion])) + emotion_beliefs_add.append(Belief(name="emotion_detected", arguments=[emotion])) except ValidationError: self.logger.warning("Invalid belief for new emotion: %s", emotion) diff --git a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py index 7bfa2e5..2527ca8 100644 --- a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py @@ -41,6 +41,8 @@ class DeepFaceEmotionRecognizer(VisualEmotionRecognizer): # Sort faces by x coordinate to maintain left-to-right order analysis.sort(key=lambda face: face['region']['x']) + # Fear op 0, boost 0.2 aan happy, sad -0.1, neutral +0.1 + analysis = [face for face in analysis if face['face_confidence'] >= 0.90] dominant_emotions = [face['dominant_emotion'] for face in analysis] diff --git a/src/control_backend/schemas/program.py b/src/control_backend/schemas/program.py index 283e17d..9bc6e0d 100644 --- a/src/control_backend/schemas/program.py +++ b/src/control_backend/schemas/program.py @@ -28,8 +28,8 @@ class LogicalOperator(Enum): OR = "OR" -type Belief = KeywordBelief | SemanticBelief | InferredBelief -type BasicBelief = KeywordBelief | SemanticBelief +type Belief = KeywordBelief | SemanticBelief | InferredBelief | EmotionBelief +type BasicBelief = KeywordBelief | SemanticBelief | EmotionBelief class KeywordBelief(ProgramElement): @@ -69,6 +69,15 @@ class InferredBelief(ProgramElement): left: Belief right: Belief +class EmotionBelief(ProgramElement): + """ + Represents a belief that is set when a certain emotion is detected. + + :ivar emotion: The emotion on which this belief gets set. + """ + + name: str = "" + emotion: str class Norm(ProgramElement): """ @@ -226,3 +235,9 @@ class Program(BaseModel): """ phases: list[Phase] + + +if __name__ == "__main__": + input = input("Enter program JSON: ") + program = Program.model_validate_json(input) + print(program) \ No newline at end of file From 985327de7058380ac545d82358603679f5ca27ed Mon Sep 17 00:00:00 2001 From: Storm Date: Mon, 19 Jan 2026 12:52:00 +0100 Subject: [PATCH 83/90] docs: updated docstrings and fixed styling ref: N25B-393 --- .../communication/ri_communication_agent.py | 2 +- .../visual_emotion_recognition_agent.py | 63 ++++++++++++------- .../visual_emotion_recognizer.py | 23 ++++--- src/control_backend/core/config.py | 7 +++ 4 files changed, 62 insertions(+), 33 deletions(-) diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 80c9a4e..e318264 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -8,7 +8,7 @@ from zmq.asyncio import Context from control_backend.agents import BaseAgent from control_backend.agents.actuation.robot_gesture_agent import RobotGestureAgent -from control_backend.agents.perception.visual_emotion_recognition_agent.visual_emotion_recognition_agent import ( +from control_backend.agents.perception.visual_emotion_recognition_agent.visual_emotion_recognition_agent import ( # noqa VisualEmotionRecognitionAgent, ) from control_backend.core.config import settings diff --git a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py index 6b11015..647ddac 100644 --- a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py @@ -4,28 +4,50 @@ from collections import Counter, defaultdict import cv2 import numpy as np -from pydantic_core import ValidationError import zmq import zmq.asyncio as azmq - -from control_backend.agents import BaseAgent -from control_backend.agents.perception.visual_emotion_recognition_agent.visual_emotion_recognizer import ( +from control_backend.agents.perception.visual_emotion_recognition_agentvisual_emotion_recognizer import ( # noqa DeepFaceEmotionRecognizer, ) +from pydantic_core import ValidationError + +from control_backend.agents import BaseAgent from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings from control_backend.schemas.belief_message import Belief -# START FROM RI COMMUNICATION AGENT? class VisualEmotionRecognitionAgent(BaseAgent): - def __init__(self, name, socket_address: str, bind: bool = False, timeout_ms: int = 1000): + def __init__(self, name: str, socket_address: str, bind: bool = False, timeout_ms: int = 1000, + window_duration: + int = settings.behaviour_settings.visual_emotion_recognition_window_duration_s + , min_frames_required: int = + settings.behaviour_settings.visual_emotion_recognition_min_frames_per_face): + """ + Initialize the Visual Emotion Recognition Agent. + + :param name: Name of the agent + :param socket_address: Address of the socket to connect or bind to + :param bind: Whether to bind to the socket address (True) or connect (False) + :param timeout_ms: Timeout for socket receive operations in milliseconds + :param window_duration: Duration in seconds over which to aggregate emotions + :param min_frames_required: Minimum number of frames per face required to consider a face + valid + """ super().__init__(name) self.socket_address = socket_address self.socket_bind = bind self.timeout_ms = timeout_ms + self.window_duration = window_duration + self.min_frames_required = min_frames_required async def setup(self): + """ + Initialize the agent resources. + 1. Initializes the :class:`VisualEmotionRecognizer`. + 2. Connects to the video input ZMQ socket. + 3. Starts the background emotion recognition loop. + """ self.logger.info("Setting up %s.", self.name) self.emotion_recognizer = DeepFaceEmotionRecognizer() @@ -45,17 +67,16 @@ class VisualEmotionRecognitionAgent(BaseAgent): async def emotion_update_loop(self): """ - Retrieve a video frame from the input socket. + Background loop to receive video frames, recognize emotions, and update beliefs. + 1. Receives video frames from the ZMQ socket. + 2. Uses the :class:`VisualEmotionRecognizer` to detect emotions. + 3. Aggregates emotions over a time window. + 4. Sends updates to the BDI Core Agent about detected emotions. """ - window_duration = 5 # seconds - next_window_time = time.time() + window_duration - - # To detect false positives - # Minimal number of frames a face has to be detected to consider it valid - # Can also reduce false positives by ignoring faces that are too small; not implemented - # Also use face confidence thresholding in recognizer - min_frames_required = 2 + # Next time to process the window and update emotions + next_window_time = time.time() + self.window_duration + # Tracks counts of detected emotions per face index face_stats = defaultdict(Counter) prev_dominant_emotions = set() @@ -82,20 +103,19 @@ class VisualEmotionRecognitionAgent(BaseAgent): # If window duration has passed, process the collected stats if time.time() >= next_window_time: - print(face_stats) window_dominant_emotions = set() # Determine dominant emotion for each face in the window for _, counter in face_stats.items(): total_detections = sum(counter.values()) - if total_detections >= min_frames_required: + if total_detections >= self.min_frames_required: dominant_emotion = counter.most_common(1)[0][0] window_dominant_emotions.add(dominant_emotion) await self.update_emotions(prev_dominant_emotions, window_dominant_emotions) prev_dominant_emotions = window_dominant_emotions face_stats.clear() - next_window_time = time.time() + window_duration + next_window_time = time.time() + self.window_duration except zmq.Again: self.logger.warning("No video frame received within timeout.") @@ -112,16 +132,15 @@ class VisualEmotionRecognitionAgent(BaseAgent): return emotion_beliefs_remove = [] - # Remove emotions that have disappeared for emotion in emotions_to_remove: self.logger.info(f"Emotion '{emotion}' has disappeared.") try: - emotion_beliefs_remove.append(Belief(name="emotion_detected", arguments=[emotion], remove=True)) + emotion_beliefs_remove.append(Belief(name="emotion_detected", arguments=[emotion], + remove=True)) except ValidationError: self.logger.warning("Invalid belief for emotion removal: %s", emotion) emotion_beliefs_add = [] - # Add new emotions that have appeared for emotion in emotions_to_add: self.logger.info(f"New emotion detected: '{emotion}'") try: @@ -131,7 +150,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): beliefs_list_add = [b.model_dump() for b in emotion_beliefs_add] beliefs_list_remove = [b.model_dump() for b in emotion_beliefs_remove] - payload = {"create": beliefs_list_add, "delete": beliefs_list_remove, "replace": []} + payload = {"create": beliefs_list_add, "delete": beliefs_list_remove} message = InternalMessage( to=settings.agent_settings.bdi_core_name, diff --git a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py index 2527ca8..89aeef3 100644 --- a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognizer.py @@ -11,20 +11,28 @@ class VisualEmotionRecognizer(abc.ABC): pass @abc.abstractmethod - def sorted_dominant_emotions(self, image): - """Recognize emotion from the given image. + def sorted_dominant_emotions(self, image) -> list[str]: + """ + Recognize dominant emotions from faces in the given image. + Emotions can be one of ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']. + To minimize false positives, consider filtering faces with low confidence. :param image: The input image for emotion recognition. - :return: Detected emotion label. + :return: List of dominant emotion detected for each face in the image, + sorted per face. """ pass class DeepFaceEmotionRecognizer(VisualEmotionRecognizer): + """ + DeepFace-based implementation of VisualEmotionRecognizer. + DeepFape has proven to be quite a pessimistic model, so expect sad, fear and neutral + emotions to be over-represented. + """ def __init__(self): self.load_model() def load_model(self): - # Initialize DeepFace model for emotion recognition print("Loading Deepface Emotion Model...") dummy_img = np.zeros((224, 224, 3), dtype=np.uint8) # analyze does not take a model as an argument, calling it once on a dummy image to load @@ -32,7 +40,7 @@ class DeepFaceEmotionRecognizer(VisualEmotionRecognizer): DeepFace.analyze(dummy_img, actions=['emotion'], enforce_detection=False) print("Deepface Emotion Model loaded.") - def sorted_dominant_emotions(self, image): + def sorted_dominant_emotions(self, image) -> list[str]: analysis = DeepFace.analyze(image, actions=['emotion'], enforce_detection=False @@ -41,12 +49,7 @@ class DeepFaceEmotionRecognizer(VisualEmotionRecognizer): # Sort faces by x coordinate to maintain left-to-right order analysis.sort(key=lambda face: face['region']['x']) - # Fear op 0, boost 0.2 aan happy, sad -0.1, neutral +0.1 - analysis = [face for face in analysis if face['face_confidence'] >= 0.90] dominant_emotions = [face['dominant_emotion'] for face in analysis] return dominant_emotions - - - diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index bacc6d4..517a924 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -78,6 +78,10 @@ class BehaviourSettings(BaseModel): :ivar transcription_words_per_token: Estimated words per token for transcription timing. :ivar transcription_token_buffer: Buffer for transcription tokens. :ivar conversation_history_length_limit: The maximum amount of messages to extract beliefs from. + :ivar visual_emotion_recognition_window_duration_s: Duration in seconds over which to aggregate + emotions and update emotion beliefs. + :ivar visual_emotion_recognition_min_frames_per_face: Minimum number of frames per face required + to consider a face valid. """ # ATTENTION: When adding/removing settings, make sure to update the .env.example file @@ -101,6 +105,9 @@ class BehaviourSettings(BaseModel): # Text belief extractor settings conversation_history_length_limit: int = 10 + # Visual Emotion Recognition settings + visual_emotion_recognition_window_duration_s: int = 5 + visual_emotion_recognition_min_frames_per_face: int = 3 class LLMSettings(BaseModel): """ From 04d19cee5cc6faaa17a5b9f51f7fbed1eb95ea2a Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Mon, 19 Jan 2026 14:08:26 +0100 Subject: [PATCH 84/90] feat: (maybe) stop response when new user message If we get a new message before the LLM is done responding, interrupt it. ref: N25B-452 --- .../agents/bdi/bdi_core_agent.py | 2 +- .../agents/bdi/text_belief_extractor_agent.py | 3 ++ src/control_backend/agents/llm/llm_agent.py | 45 ++++++++++++++----- src/control_backend/core/config.py | 1 + 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 628bb53..685a3b6 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -338,7 +338,7 @@ class BDICoreAgent(BaseAgent): yield @self.actions.add(".reply_with_goal", 3) - def _reply_with_goal(agent: "BDICoreAgent", term, intention): + def _reply_with_goal(agent, term, intention): """ Let the LLM generate a response to a user's utterance with the current norms and a specific goal. diff --git a/src/control_backend/agents/bdi/text_belief_extractor_agent.py b/src/control_backend/agents/bdi/text_belief_extractor_agent.py index 362dfbf..9ea6b9a 100644 --- a/src/control_backend/agents/bdi/text_belief_extractor_agent.py +++ b/src/control_backend/agents/bdi/text_belief_extractor_agent.py @@ -318,6 +318,9 @@ class TextBeliefExtractorAgent(BaseAgent): async with httpx.AsyncClient() as client: response = await client.post( settings.llm_settings.local_llm_url, + headers={"Authorization": f"Bearer {settings.llm_settings.api_key}"} + if settings.llm_settings.api_key + else {}, json={ "model": settings.llm_settings.local_llm_model, "messages": [{"role": "user", "content": prompt}], diff --git a/src/control_backend/agents/llm/llm_agent.py b/src/control_backend/agents/llm/llm_agent.py index 1c72dfc..7cac097 100644 --- a/src/control_backend/agents/llm/llm_agent.py +++ b/src/control_backend/agents/llm/llm_agent.py @@ -1,3 +1,4 @@ +import asyncio import json import re import uuid @@ -32,6 +33,9 @@ class LLMAgent(BaseAgent): def __init__(self, name: str): super().__init__(name) self.history = [] + self._querying = False + self._interrupted = False + self._go_ahead = asyncio.Event() async def setup(self): self.logger.info("Setting up %s.", self.name) @@ -50,7 +54,7 @@ class LLMAgent(BaseAgent): case "prompt_message": try: prompt_message = LLMPromptMessage.model_validate_json(msg.body) - await self._process_bdi_message(prompt_message) + self.add_behavior(self._process_bdi_message(prompt_message)) # no block except ValidationError: self.logger.debug("Prompt message from BDI core is invalid.") case "assistant_message": @@ -73,12 +77,35 @@ class LLMAgent(BaseAgent): :param message: The parsed prompt message containing text, norms, and goals. """ + if self._querying: + self.logger.debug("Received another BDI prompt while processing previous message.") + self._interrupted = True # interrupt the previous processing + await self._go_ahead.wait() # wait until we get the go-ahead + + self._go_ahead.clear() + self._querying = True full_message = "" async for chunk in self._query_llm(message.text, message.norms, message.goals): + if self._interrupted: + self.logger.debug("Interrupted processing of previous message.") + break await self._send_reply(chunk) full_message += chunk - self.logger.debug("Finished processing BDI message. Response sent in chunks to BDI core.") - await self._send_full_reply(full_message) + else: + self._querying = False + + self.history.append( + { + "role": "assistant", + "content": full_message, + } + ) + self.logger.debug( + "Finished processing BDI message. Response sent in chunks to BDI core." + ) + await self._send_full_reply(full_message) + + self._interrupted = False async def _send_reply(self, msg: str): """ @@ -141,7 +168,7 @@ class LLMAgent(BaseAgent): full_message += token current_chunk += token - self.logger.llm( + self.logger.debug( "Received token: %s", full_message, extra={"reference": message_id}, # Used in the UI to update old logs @@ -159,13 +186,6 @@ class LLMAgent(BaseAgent): # Yield any remaining tail if current_chunk: yield current_chunk - - self.history.append( - { - "role": "assistant", - "content": full_message, - } - ) except httpx.HTTPError as err: self.logger.error("HTTP error.", exc_info=err) yield "LLM service unavailable." @@ -185,6 +205,9 @@ class LLMAgent(BaseAgent): async with client.stream( "POST", settings.llm_settings.local_llm_url, + headers={"Authorization": f"Bearer {settings.llm_settings.api_key}"} + if settings.llm_settings.api_key + else {}, json={ "model": settings.llm_settings.local_llm_model, "messages": messages, diff --git a/src/control_backend/core/config.py b/src/control_backend/core/config.py index 329a246..82b9ede 100644 --- a/src/control_backend/core/config.py +++ b/src/control_backend/core/config.py @@ -117,6 +117,7 @@ class LLMSettings(BaseModel): local_llm_url: str = "http://localhost:1234/v1/chat/completions" local_llm_model: str = "gpt-oss" + api_key: str = "" chat_temperature: float = 1.0 code_temperature: float = 0.3 n_parallel: int = 4 From c0789e82a985efe6867dc7df029d170594d0de97 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Mon, 19 Jan 2026 14:47:11 +0100 Subject: [PATCH 85/90] feat: add previously interrupted message to current ref: N25B-452 --- src/control_backend/agents/llm/llm_agent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/control_backend/agents/llm/llm_agent.py b/src/control_backend/agents/llm/llm_agent.py index 7cac097..ca0cd78 100644 --- a/src/control_backend/agents/llm/llm_agent.py +++ b/src/control_backend/agents/llm/llm_agent.py @@ -35,6 +35,7 @@ class LLMAgent(BaseAgent): self.history = [] self._querying = False self._interrupted = False + self._interrupted_message = "" self._go_ahead = asyncio.Event() async def setup(self): @@ -82,11 +83,14 @@ class LLMAgent(BaseAgent): self._interrupted = True # interrupt the previous processing await self._go_ahead.wait() # wait until we get the go-ahead + message.text = f"{self._interrupted_message} {message.text}" + self._go_ahead.clear() self._querying = True full_message = "" async for chunk in self._query_llm(message.text, message.norms, message.goals): if self._interrupted: + self._interrupted_message = message self.logger.debug("Interrupted processing of previous message.") break await self._send_reply(chunk) @@ -105,6 +109,7 @@ class LLMAgent(BaseAgent): ) await self._send_full_reply(full_message) + self._go_ahead.set() self._interrupted = False async def _send_reply(self, msg: str): From 1cd5b46f9743c823c40b7f2d8e484b03d4e4f33e Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Mon, 19 Jan 2026 15:03:59 +0100 Subject: [PATCH 86/90] fix: should work now Also added trimming to Windows transcription. ref: N25B-452 --- src/control_backend/agents/llm/llm_agent.py | 4 ++-- .../perception/transcription_agent/speech_recognizer.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/control_backend/agents/llm/llm_agent.py b/src/control_backend/agents/llm/llm_agent.py index ca0cd78..db7e363 100644 --- a/src/control_backend/agents/llm/llm_agent.py +++ b/src/control_backend/agents/llm/llm_agent.py @@ -90,7 +90,7 @@ class LLMAgent(BaseAgent): full_message = "" async for chunk in self._query_llm(message.text, message.norms, message.goals): if self._interrupted: - self._interrupted_message = message + self._interrupted_message = message.text self.logger.debug("Interrupted processing of previous message.") break await self._send_reply(chunk) @@ -173,7 +173,7 @@ class LLMAgent(BaseAgent): full_message += token current_chunk += token - self.logger.debug( + self.logger.llm( "Received token: %s", full_message, extra={"reference": message_id}, # Used in the UI to update old logs diff --git a/src/control_backend/agents/perception/transcription_agent/speech_recognizer.py b/src/control_backend/agents/perception/transcription_agent/speech_recognizer.py index 9fae676..1fe7e3f 100644 --- a/src/control_backend/agents/perception/transcription_agent/speech_recognizer.py +++ b/src/control_backend/agents/perception/transcription_agent/speech_recognizer.py @@ -145,4 +145,6 @@ class OpenAIWhisperSpeechRecognizer(SpeechRecognizer): def recognize_speech(self, audio: np.ndarray) -> str: self.load_model() - return whisper.transcribe(self.model, audio, **self._get_decode_options(audio))["text"] + return whisper.transcribe(self.model, audio, **self._get_decode_options(audio))[ + "text" + ].strip() From 230afef16fe5630086aa9cfc609064dec38f7464 Mon Sep 17 00:00:00 2001 From: Kasper Marinus Date: Mon, 19 Jan 2026 16:06:17 +0100 Subject: [PATCH 87/90] test: fix tests ref: N25B-452 --- .../agents/bdi/bdi_core_agent.py | 4 -- src/control_backend/agents/llm/llm_agent.py | 12 +++- test/unit/agents/llm/test_llm_agent.py | 66 +++++++++++++------ 3 files changed, 55 insertions(+), 27 deletions(-) diff --git a/src/control_backend/agents/bdi/bdi_core_agent.py b/src/control_backend/agents/bdi/bdi_core_agent.py index 685a3b6..54b5149 100644 --- a/src/control_backend/agents/bdi/bdi_core_agent.py +++ b/src/control_backend/agents/bdi/bdi_core_agent.py @@ -512,10 +512,6 @@ class BDICoreAgent(BaseAgent): yield - @self.actions.add(".notify_ui", 0) - def _notify_ui(agent, term, intention): - pass - async def _send_to_llm(self, text: str, norms: str, goals: str): """ Sends a text query to the LLM agent asynchronously. diff --git a/src/control_backend/agents/llm/llm_agent.py b/src/control_backend/agents/llm/llm_agent.py index db7e363..8d81249 100644 --- a/src/control_backend/agents/llm/llm_agent.py +++ b/src/control_backend/agents/llm/llm_agent.py @@ -59,9 +59,9 @@ class LLMAgent(BaseAgent): except ValidationError: self.logger.debug("Prompt message from BDI core is invalid.") case "assistant_message": - self.history.append({"role": "assistant", "content": msg.body}) + self._apply_conversation_message({"role": "assistant", "content": msg.body}) case "user_message": - self.history.append({"role": "user", "content": msg.body}) + self._apply_conversation_message({"role": "user", "content": msg.body}) elif msg.sender == settings.agent_settings.bdi_program_manager_name: if msg.body == "clear_history": self.logger.debug("Clearing conversation history.") @@ -98,7 +98,7 @@ class LLMAgent(BaseAgent): else: self._querying = False - self.history.append( + self._apply_conversation_message( { "role": "assistant", "content": full_message, @@ -112,6 +112,12 @@ class LLMAgent(BaseAgent): self._go_ahead.set() self._interrupted = False + def _apply_conversation_message(self, message: dict[str, str]): + if len(self.history) > 0 and message["role"] == self.history[-1]["role"]: + self.history[-1]["content"] += " " + message["content"] + return + self.history.append(message) + async def _send_reply(self, msg: str): """ Sends a response message (chunk) back to the BDI Core Agent. diff --git a/test/unit/agents/llm/test_llm_agent.py b/test/unit/agents/llm/test_llm_agent.py index a1cc297..bd407cc 100644 --- a/test/unit/agents/llm/test_llm_agent.py +++ b/test/unit/agents/llm/test_llm_agent.py @@ -61,8 +61,52 @@ async def test_llm_processing_success(mock_httpx_client, mock_settings): thread="prompt_message", # REQUIRED: thread must match handle_message logic ) + agent._process_bdi_message = AsyncMock() + await agent.handle_message(msg) + agent._process_bdi_message.assert_called() + + +@pytest.mark.asyncio +async def test_process_bdi_message_success(mock_httpx_client, mock_settings): + # Setup the mock response for the stream + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + + # Simulate stream lines + lines = [ + b'data: {"choices": [{"delta": {"content": "Hello"}}]}', + b'data: {"choices": [{"delta": {"content": " world"}}]}', + b'data: {"choices": [{"delta": {"content": "."}}]}', + b"data: [DONE]", + ] + + async def aiter_lines_gen(): + for line in lines: + yield line.decode() + + mock_response.aiter_lines.side_effect = aiter_lines_gen + + mock_stream_context = MagicMock() + mock_stream_context.__aenter__ = AsyncMock(return_value=mock_response) + mock_stream_context.__aexit__ = AsyncMock(return_value=None) + + # Configure the client + mock_httpx_client.stream = MagicMock(return_value=mock_stream_context) + + # Setup Agent + agent = LLMAgent("llm_agent") + agent.send = AsyncMock() # Mock the send method to verify replies + + mock_logger = MagicMock() + agent.logger = mock_logger + + # Simulate receiving a message from BDI + prompt = LLMPromptMessage(text="Hi", norms=[], goals=[]) + + await agent._process_bdi_message(prompt) + # Verification # "Hello world." constitutes one sentence/chunk based on punctuation split # The agent should call send once with the full sentence, PLUS once more for full reply @@ -79,28 +123,16 @@ async def test_llm_processing_errors(mock_httpx_client, mock_settings): agent = LLMAgent("llm_agent") agent.send = AsyncMock() prompt = LLMPromptMessage(text="Hi", norms=[], goals=[]) - msg = InternalMessage( - to="llm", - sender=mock_settings.agent_settings.bdi_core_name, - body=prompt.model_dump_json(), - thread="prompt_message", - ) # HTTP Error: stream method RAISES exception immediately mock_httpx_client.stream = MagicMock(side_effect=httpx.HTTPError("Fail")) - await agent.handle_message(msg) + await agent._process_bdi_message(prompt) # Check that error message was sent assert agent.send.called assert "LLM service unavailable." in agent.send.call_args_list[0][0][0].body - # General Exception - agent.send.reset_mock() - mock_httpx_client.stream = MagicMock(side_effect=Exception("Boom")) - await agent.handle_message(msg) - assert "Error processing the request." in agent.send.call_args_list[0][0][0].body - @pytest.mark.asyncio async def test_llm_json_error(mock_httpx_client, mock_settings): @@ -125,13 +157,7 @@ async def test_llm_json_error(mock_httpx_client, mock_settings): agent.logger = MagicMock() prompt = LLMPromptMessage(text="Hi", norms=[], goals=[]) - msg = InternalMessage( - to="llm", - sender=mock_settings.agent_settings.bdi_core_name, - body=prompt.model_dump_json(), - thread="prompt_message", - ) - await agent.handle_message(msg) + await agent._process_bdi_message(prompt) agent.logger.error.assert_called() # Should log JSONDecodeError From bc0947fac18de7220dd3a738c8e8f1db8a8fdb2e Mon Sep 17 00:00:00 2001 From: Pim Hutting Date: Mon, 19 Jan 2026 18:26:15 +0100 Subject: [PATCH 88/90] chore: added a dot --- .../visual_emotion_recognition_agent.py | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py index 647ddac..465903b 100644 --- a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py @@ -6,23 +6,27 @@ import cv2 import numpy as np import zmq import zmq.asyncio as azmq -from control_backend.agents.perception.visual_emotion_recognition_agentvisual_emotion_recognizer import ( # noqa - DeepFaceEmotionRecognizer, -) from pydantic_core import ValidationError from control_backend.agents import BaseAgent +from control_backend.agents.perception.visual_emotion_recognition_agent.visual_emotion_recognizer import ( # noqa + DeepFaceEmotionRecognizer, +) from control_backend.core.agent_system import InternalMessage from control_backend.core.config import settings from control_backend.schemas.belief_message import Belief class VisualEmotionRecognitionAgent(BaseAgent): - def __init__(self, name: str, socket_address: str, bind: bool = False, timeout_ms: int = 1000, - window_duration: - int = settings.behaviour_settings.visual_emotion_recognition_window_duration_s - , min_frames_required: int = - settings.behaviour_settings.visual_emotion_recognition_min_frames_per_face): + def __init__( + self, + name: str, + socket_address: str, + bind: bool = False, + timeout_ms: int = 1000, + window_duration: int = settings.behaviour_settings.visual_emotion_recognition_window_duration_s, # noqa + min_frames_required: int = settings.behaviour_settings.visual_emotion_recognition_min_frames_per_face, # noqa + ): """ Initialize the Visual Emotion Recognition Agent. @@ -31,7 +35,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): :param bind: Whether to bind to the socket address (True) or connect (False) :param timeout_ms: Timeout for socket receive operations in milliseconds :param window_duration: Duration in seconds over which to aggregate emotions - :param min_frames_required: Minimum number of frames per face required to consider a face + :param min_frames_required: Minimum number of frames per face required to consider a face valid """ super().__init__(name) @@ -78,29 +82,29 @@ class VisualEmotionRecognitionAgent(BaseAgent): # Tracks counts of detected emotions per face index face_stats = defaultdict(Counter) - + prev_dominant_emotions = set() while self._running: try: frame_bytes = await self.video_in_socket.recv() - + # Convert bytes to a numpy buffer nparr = np.frombuffer(frame_bytes, np.uint8) - + # Decode image into the generic Numpy Array DeepFace expects frame_image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) if frame_image is None: # Could not decode image, skip this frame continue - + # Get the dominant emotion from each face current_emotions = self.emotion_recognizer.sorted_dominant_emotions(frame_image) # Update emotion counts for each detected face for i, emotion in enumerate(current_emotions): face_stats[i][emotion] += 1 - + # If window duration has passed, process the collected stats if time.time() >= next_window_time: window_dominant_emotions = set() @@ -111,35 +115,36 @@ class VisualEmotionRecognitionAgent(BaseAgent): if total_detections >= self.min_frames_required: dominant_emotion = counter.most_common(1)[0][0] window_dominant_emotions.add(dominant_emotion) - + await self.update_emotions(prev_dominant_emotions, window_dominant_emotions) prev_dominant_emotions = window_dominant_emotions face_stats.clear() next_window_time = time.time() + self.window_duration - + except zmq.Again: self.logger.warning("No video frame received within timeout.") async def update_emotions(self, prev_emotions: set[str], emotions: set[str]): """ - Compare emotions from previous window and current emotions, + Compare emotions from previous window and current emotions, send updates to BDI Core Agent. """ emotions_to_remove = prev_emotions - emotions emotions_to_add = emotions - prev_emotions if not emotions_to_add and not emotions_to_remove: - return - + return + emotion_beliefs_remove = [] for emotion in emotions_to_remove: self.logger.info(f"Emotion '{emotion}' has disappeared.") try: - emotion_beliefs_remove.append(Belief(name="emotion_detected", arguments=[emotion], - remove=True)) + emotion_beliefs_remove.append( + Belief(name="emotion_detected", arguments=[emotion], remove=True) + ) except ValidationError: self.logger.warning("Invalid belief for emotion removal: %s", emotion) - + emotion_beliefs_add = [] for emotion in emotions_to_add: self.logger.info(f"New emotion detected: '{emotion}'") @@ -147,7 +152,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): emotion_beliefs_add.append(Belief(name="emotion_detected", arguments=[emotion])) except ValidationError: self.logger.warning("Invalid belief for new emotion: %s", emotion) - + beliefs_list_add = [b.model_dump() for b in emotion_beliefs_add] beliefs_list_remove = [b.model_dump() for b in emotion_beliefs_remove] payload = {"create": beliefs_list_add, "delete": beliefs_list_remove} @@ -158,4 +163,4 @@ class VisualEmotionRecognitionAgent(BaseAgent): body=json.dumps(payload), thread="beliefs", ) - await self.send(message) \ No newline at end of file + await self.send(message) From f9b807fc971829bf96577b4f959b8673d9baa5fe Mon Sep 17 00:00:00 2001 From: Storm Date: Tue, 20 Jan 2026 12:46:30 +0100 Subject: [PATCH 89/90] chore: quick push before demo; fixed image receiving from RI --- .../agents/communication/ri_communication_agent.py | 6 ++++++ .../visual_emotion_recognition_agent.py | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index e318264..2056c0d 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -55,6 +55,7 @@ class RICommunicationAgent(BaseAgent): self.connected = False self.gesture_agent: RobotGestureAgent | None = None self.speech_agent: RobotSpeechAgent | None = None + self.visual_emotion_recognition_agent: VisualEmotionRecognitionAgent | None = None async def setup(self): """ @@ -218,6 +219,7 @@ class RICommunicationAgent(BaseAgent): socket_address=addr, bind=bind, ) + self.visual_emotion_recognition_agent = visual_emotion_agent await visual_emotion_agent.start() case _: self.logger.warning("Unhandled negotiation id: %s", id) @@ -323,6 +325,9 @@ class RICommunicationAgent(BaseAgent): if self.speech_agent is not None: await self.speech_agent.stop() + + if self.visual_emotion_recognition_agent is not None: + await self.visual_emotion_recognition_agent.stop() if self.pub_socket is not None: self.pub_socket.close() @@ -332,6 +337,7 @@ class RICommunicationAgent(BaseAgent): self.connected = True async def handle_message(self, msg: InternalMessage): + return try: pause_command = PauseCommand.model_validate_json(msg.body) await self._req_socket.send_json(pause_command.model_dump()) diff --git a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py index 465903b..5344b9b 100644 --- a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py @@ -7,6 +7,7 @@ import numpy as np import zmq import zmq.asyncio as azmq from pydantic_core import ValidationError +import struct from control_backend.agents import BaseAgent from control_backend.agents.perception.visual_emotion_recognition_agent.visual_emotion_recognizer import ( # noqa @@ -88,7 +89,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): while self._running: try: frame_bytes = await self.video_in_socket.recv() - + # Convert bytes to a numpy buffer nparr = np.frombuffer(frame_bytes, np.uint8) @@ -97,6 +98,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): if frame_image is None: # Could not decode image, skip this frame + self.logger.warning("Received invalid video frame, skipping.") continue # Get the dominant emotion from each face @@ -124,6 +126,7 @@ class VisualEmotionRecognitionAgent(BaseAgent): except zmq.Again: self.logger.warning("No video frame received within timeout.") + async def update_emotions(self, prev_emotions: set[str], emotions: set[str]): """ Compare emotions from previous window and current emotions, From 0b1c2ce20a6f00c06d0e65b01044e8412745a10c Mon Sep 17 00:00:00 2001 From: Storm Date: Tue, 20 Jan 2026 18:53:24 +0100 Subject: [PATCH 90/90] feat: implemented pausing, implemented graceful stopping, removed old RI pausing code ref: N25B-393 --- .../communication/ri_communication_agent.py | 13 +----- .../visual_emotion_recognition_agent.py | 40 ++++++++++++++++++- .../user_interrupt/user_interrupt_agent.py | 29 ++++++-------- 3 files changed, 52 insertions(+), 30 deletions(-) diff --git a/src/control_backend/agents/communication/ri_communication_agent.py b/src/control_backend/agents/communication/ri_communication_agent.py index 2056c0d..746705c 100644 --- a/src/control_backend/agents/communication/ri_communication_agent.py +++ b/src/control_backend/agents/communication/ri_communication_agent.py @@ -3,7 +3,6 @@ import json import zmq import zmq.asyncio as azmq -from pydantic import ValidationError from zmq.asyncio import Context from control_backend.agents import BaseAgent @@ -12,8 +11,6 @@ from control_backend.agents.perception.visual_emotion_recognition_agent.visual_e VisualEmotionRecognitionAgent, ) from control_backend.core.config import settings -from control_backend.schemas.internal_message import InternalMessage -from control_backend.schemas.ri_message import PauseCommand from ..actuation.robot_speech_agent import RobotSpeechAgent from ..perception import VADAgent @@ -335,12 +332,4 @@ class RICommunicationAgent(BaseAgent): self.logger.debug("Restarting communication negotiation.") if await self._negotiate_connection(max_retries=2): self.connected = True - - async def handle_message(self, msg: InternalMessage): - return - try: - pause_command = PauseCommand.model_validate_json(msg.body) - await self._req_socket.send_json(pause_command.model_dump()) - self.logger.debug(await self._req_socket.recv_json()) - except ValidationError: - self.logger.warning("Incorrect message format for PauseCommand.") + \ No newline at end of file diff --git a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py index 5344b9b..52f97a2 100644 --- a/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py +++ b/src/control_backend/agents/perception/visual_emotion_recognition_agent/visual_emotion_recognition_agent.py @@ -1,3 +1,4 @@ +import asyncio import json import time from collections import Counter, defaultdict @@ -7,7 +8,6 @@ import numpy as np import zmq import zmq.asyncio as azmq from pydantic_core import ValidationError -import struct from control_backend.agents import BaseAgent from control_backend.agents.perception.visual_emotion_recognition_agent.visual_emotion_recognizer import ( # noqa @@ -46,6 +46,11 @@ class VisualEmotionRecognitionAgent(BaseAgent): self.window_duration = window_duration self.min_frames_required = min_frames_required + # Pause functionality + # NOTE: flag is set when running, cleared when paused + self._paused = asyncio.Event() + self._paused.set() + async def setup(self): """ Initialize the agent resources. @@ -88,6 +93,8 @@ class VisualEmotionRecognitionAgent(BaseAgent): while self._running: try: + await self._paused.wait() + frame_bytes = await self.video_in_socket.recv() # Convert bytes to a numpy buffer @@ -167,3 +174,34 @@ class VisualEmotionRecognitionAgent(BaseAgent): thread="beliefs", ) await self.send(message) + + async def handle_message(self, msg: InternalMessage): + """ + Handle incoming messages. + + Expects messages to pause or resume the Visual Emotion Recognition + processing from User Interrupt Agent. + + :param msg: The received internal message. + """ + sender = msg.sender + + if sender == settings.agent_settings.user_interrupt_name: + if msg.body == "PAUSE": + self.logger.info("Pausing Visual Emotion Recognition processing.") + self._paused.clear() + elif msg.body == "RESUME": + self.logger.info("Resuming Visual Emotion Recognition processing.") + self._paused.set() + else: + self.logger.warning(f"Unknown command from User Interrupt Agent: {msg.body}") + else: + self.logger.debug(f"Ignoring message from unknown sender: {sender}") + + + async def stop(self): + """ + Clean up resources used by the agent. + """ + self.video_in_socket.close() + await super().stop() \ No newline at end of file diff --git a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py index a42861a..1454c44 100644 --- a/src/control_backend/agents/user_interrupt/user_interrupt_agent.py +++ b/src/control_backend/agents/user_interrupt/user_interrupt_agent.py @@ -378,34 +378,29 @@ class UserInterruptAgent(BaseAgent): self.logger.debug("Sending experiment control '%s' to BDI Core.", thread) await self.send(out_msg) - async def _send_pause_command(self, pause): + async def _send_pause_command(self, pause: str): """ - Send a pause command to the Robot Interface via the RI Communication Agent. - Send a pause command to the other internal agents; for now just VAD agent. + Send a pause command to the other internal agents; for now just VAD and VED agent. """ - cmd = PauseCommand(data=pause) - message = InternalMessage( - to=settings.agent_settings.ri_communication_name, - sender=self.name, - body=cmd.model_dump_json(), - ) - await self.send(message) - if pause == "true": - # Send pause to VAD agent + # Send pause to VAD and VED agent vad_message = InternalMessage( - to=settings.agent_settings.vad_name, + to=[settings.agent_settings.vad_name, + settings.agent_settings.visual_emotion_recognition_name], sender=self.name, body="PAUSE", ) await self.send(vad_message) - self.logger.info("Sent pause command to VAD Agent and RI Communication Agent.") + # Voice Activity Detection and Visual Emotion Recognition agents + self.logger.info("Sent pause command to VAD and VED agents.") else: - # Send resume to VAD agent + # Send resume to VAD and VED agents vad_message = InternalMessage( - to=settings.agent_settings.vad_name, + to=[settings.agent_settings.vad_name, + settings.agent_settings.visual_emotion_recognition_name], sender=self.name, body="RESUME", ) await self.send(vad_message) - self.logger.info("Sent resume command to VAD Agent and RI Communication Agent.") + # Voice Activity Detection and Visual Emotion Recognition agents + self.logger.info("Sent resume command to VAD and VED agents.") \ No newline at end of file