From c577daa9e6d750f730b85ade0e4978072f2c4179 Mon Sep 17 00:00:00 2001
From: Kasper <kasper@kasper-deb.amsk.lan>
Date: Fri, 26 Sep 2025 21:39:11 +0200
Subject: [PATCH 1/4] feat: add basic UI2CB and CB2UI communication

By pressing the button, the text in the input field is sent to the CB.

Every second, UI receives the current time from CB.

ref: N25B-107
ref: N25B-110
---
 src/App.tsx | 68 ++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 44 insertions(+), 24 deletions(-)

diff --git a/src/App.tsx b/src/App.tsx
index 3d7ded3..e9ce59b 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -1,35 +1,55 @@
-import { useState } from 'react'
-import reactLogo from './assets/react.svg'
-import viteLogo from '/vite.svg'
+import { useState, useEffect } from 'react'
 import './App.css'
 
 function App() {
-  const [count, setCount] = useState(0)
+  const [message, setMessage] = useState('');
+  const [sseMessage, setSseMessage] = useState('');
+
+  const sendMessage = async () => {
+    try {
+      const response = await fetch("http://localhost:8000/message", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ message }),
+      });
+      const data = await response.json();
+      console.log(data);
+    } catch (error) {
+      console.error("Error sending message: ", error);
+    }
+  };
+
+  useEffect(() => {
+    const eventSource = new EventSource("http://localhost:8000/sse");
+
+    eventSource.onmessage = (event) => {
+      setSseMessage(event.data);
+    };
+
+    return () => {
+      eventSource.close();
+    };
+  });
 
   return (
-    <>
+    <div className="App">
       <div>
-        <a href="https://vite.dev" target="_blank">
-          <img src={viteLogo} className="logo" alt="Vite logo" />
-        </a>
-        <a href="https://react.dev" target="_blank">
-          <img src={reactLogo} className="logo react" alt="React logo" />
-        </a>
+        <input
+          type="text"
+          value={message}
+          onChange={(e) => setMessage(e.target.value)}
+          placeholder="Enter a message"
+        />
+        <button onClick={sendMessage}>Send Message to Backend</button>
       </div>
-      <h1>Vite + React</h1>
-      <div className="card">
-        <button onClick={() => setCount((count) => count + 1)}>
-          count is {count}
-        </button>
-        <p>
-          Edit <code>src/App.tsx</code> and save to test HMR
-        </p>
+      <div>
+        <h2>Message from Server (SSE):</h2>
+        <p>{sseMessage}</p>
       </div>
-      <p className="read-the-docs">
-        Click on the Vite and React logos to learn more
-      </p>
-    </>
-  )
+    </div>
+  );
 }
 
 export default App

From c512739a25b29f9631db23014f7a0dd40ac33496 Mon Sep 17 00:00:00 2001
From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com>
Date: Wed, 1 Oct 2025 10:17:16 +0200
Subject: [PATCH 2/4] feat: differentiate between SSE messages

For spoken text, we have JSON data that can be differentiated from other data. We show this spoken text in a different UI field.

ref: N25B-110
---
 src/App.tsx | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/App.tsx b/src/App.tsx
index e9ce59b..53505d5 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -4,6 +4,7 @@ import './App.css'
 function App() {
   const [message, setMessage] = useState('');
   const [sseMessage, setSseMessage] = useState('');
+  const [spoken, setSpoken] = useState<string>("");
 
   const sendMessage = async () => {
     try {
@@ -26,12 +27,17 @@ function App() {
 
     eventSource.onmessage = (event) => {
       setSseMessage(event.data);
+
+      try {
+          const data = JSON.parse(event.data);
+          if (data.speech) setSpoken(data.speech);
+      } catch {}
     };
 
     return () => {
       eventSource.close();
     };
-  });
+  }, []);
 
   return (
     <div className="App">
@@ -40,6 +46,7 @@ function App() {
           type="text"
           value={message}
           onChange={(e) => setMessage(e.target.value)}
+          onKeyDown={(e) => e.key === "Enter" && sendMessage().then(() => setMessage(""))}
           placeholder="Enter a message"
         />
         <button onClick={sendMessage}>Send Message to Backend</button>
@@ -48,6 +55,10 @@ function App() {
         <h2>Message from Server (SSE):</h2>
         <p>{sseMessage}</p>
       </div>
+      <div>
+        <h2>Spoken text (SSE):</h2>
+        <p>{spoken}</p>
+      </div>
     </div>
   );
 }

From 96053e798ad07f0eaa6415de75f41bbfee453f09 Mon Sep 17 00:00:00 2001
From: JobvAlewijk <extrajvanalewijk@gmail.com>
Date: Wed, 1 Oct 2025 14:06:30 +0200
Subject: [PATCH 3/4] fix: moved ui2cb communication into server

---
 src/App.tsx                           | 68 ++---------------------
 src/pages/Home/Home.tsx               |  4 +-
 src/pages/ServerComms/ServerComms.css |  0
 src/pages/ServerComms/ServerComms.tsx | 80 +++++++++++++++++++++++++++
 src/pages/TemplatePage/Template.tsx   |  3 +-
 5 files changed, 88 insertions(+), 67 deletions(-)
 create mode 100644 src/pages/ServerComms/ServerComms.css
 create mode 100644 src/pages/ServerComms/ServerComms.tsx

diff --git a/src/App.tsx b/src/App.tsx
index 8c9d79d..757e769 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -1,78 +1,18 @@
-import { useState, useEffect } from 'react'
 import { Routes, Route } from 'react-router'
 import './App.css'
 import TemplatePage from './pages/TemplatePage/Template.tsx'
 import Home from './pages/Home/Home.tsx'
+import ServerComms from './pages/ServerComms/ServerComms.tsx'
 
-function App() {
-  const [message, setMessage] = useState('');
-  const [sseMessage, setSseMessage] = useState('');
-  const [spoken, setSpoken] = useState<string>("");
-
-  const sendMessage = async () => {
-    try {
-      const response = await fetch("http://localhost:8000/message", {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify({ message }),
-      });
-      const data = await response.json();
-      console.log(data);
-    } catch (error) {
-      console.error("Error sending message: ", error);
-    }
-  };
-
-  useEffect(() => {
-    const eventSource = new EventSource("http://localhost:8000/sse");
-
-    eventSource.onmessage = (event) => {
-      setSseMessage(event.data);
-
-      try {
-          const data = JSON.parse(event.data);
-          if (data.speech) setSpoken(data.speech);
-      } catch {}
-    };
-
-    return () => {
-      eventSource.close();
-    };
-  }, []);
-
-  return (
-    <div className="App">
-      <div>
-        <input
-          type="text"
-          value={message}
-          onChange={(e) => setMessage(e.target.value)}
-          onKeyDown={(e) => e.key === "Enter" && sendMessage().then(() => setMessage(""))}
-          placeholder="Enter a message"
-        />
-        <button onClick={sendMessage}>Send Message to Backend</button>
-      </div>
-      <div>
-        <h2>Message from Server (SSE):</h2>
-        <p>{sseMessage}</p>
-      </div>
-      <div>
-        <h2>Spoken text (SSE):</h2>
-        <p>{spoken}</p>
-      </div>
-    </div>
-  );
-/*
-  function App(){
+function App(){
   
   return (
     <Routes>
       <Route path="/" element={<Home />} />
       <Route path="/Template" element={<TemplatePage />} />
+      <Route path="/ServerComms" element = {<ServerComms />} />
     </Routes>
   )
 }
-*/
+
 export default App
diff --git a/src/pages/Home/Home.tsx b/src/pages/Home/Home.tsx
index 5c4a72d..8767db3 100644
--- a/src/pages/Home/Home.tsx
+++ b/src/pages/Home/Home.tsx
@@ -1,4 +1,4 @@
-import { useState } from 'react'
+//import { useState } from 'react'
 import { Link } from 'react-router'
 import reactLogo from '../../assets/react.svg'
 import viteLogo from '../../assets/vite.svg'
@@ -29,7 +29,7 @@ function Home() {
       <h1>Vite + React</h1>
       
         <Counter />
-        <Link to = '/Template'>
+        <Link to = '/ServerComms'>
             <button className='movePage right' onClick={() : void => {}}>
             Page Cool --{'>'}
             </button>
diff --git a/src/pages/ServerComms/ServerComms.css b/src/pages/ServerComms/ServerComms.css
new file mode 100644
index 0000000..e69de29
diff --git a/src/pages/ServerComms/ServerComms.tsx b/src/pages/ServerComms/ServerComms.tsx
new file mode 100644
index 0000000..c16ec81
--- /dev/null
+++ b/src/pages/ServerComms/ServerComms.tsx
@@ -0,0 +1,80 @@
+import { useState, useEffect } from 'react'
+import { Link } from 'react-router'
+//import Counter from '../../components/components.tsx'
+
+
+//this is your css file where you can style your buttons and such
+//you can still use css parts from App.css, but also overwrite them
+
+function ServerComms() {
+  const [message, setMessage] = useState('');
+  const [sseMessage, setSseMessage] = useState('');
+  const [spoken, setSpoken] = useState<string>("");
+
+  const sendMessage = async () => {
+    try {
+      const response = await fetch("http://localhost:8000/message", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ message }),
+      });
+      const data = await response.json();
+      console.log(data);
+    } catch (error) {
+      console.error("Error sending message: ", error);
+    }
+  };
+
+  useEffect(() => {
+    const eventSource = new EventSource("http://localhost:8000/sse");
+
+    eventSource.onmessage = (event) => {
+      setSseMessage(event.data);
+
+      try {
+          const data = JSON.parse(event.data);
+          if (data.speech) setSpoken(data.speech);
+      } catch {}
+    };
+
+    return () => {
+      eventSource.close();
+    };
+  }, []);
+
+  return (
+    <div className="App">
+      <div>
+        <input
+          type="text"
+          value={message}
+          onChange={(e) => setMessage(e.target.value)}
+          onKeyDown={(e) => e.key === "Enter" && sendMessage().then(() => setMessage(""))}
+          placeholder="Enter a message"
+        />
+        <button onClick={sendMessage}>Send Message to Backend</button>
+      </div>
+      <div>
+        <h2>Message from Server (SSE):</h2>
+        <p>{sseMessage}</p>
+      </div>
+      <div>
+        <h2>Spoken text (SSE):</h2>
+        <p>{spoken}</p>
+      </div>
+      <div>
+        <Link to = {"/"}> {/* here you link to the homepage, in App.tsx you can link new pages */}
+          <button className= 'movePage left' onClick={() :void => {}}>
+            Page {'<'}-- Go Home
+          </button>
+        </Link>
+      </div>
+    </div>
+    
+    
+  );
+}
+
+export default ServerComms
\ No newline at end of file
diff --git a/src/pages/TemplatePage/Template.tsx b/src/pages/TemplatePage/Template.tsx
index 4cb3118..1a3feac 100644
--- a/src/pages/TemplatePage/Template.tsx
+++ b/src/pages/TemplatePage/Template.tsx
@@ -1,7 +1,8 @@
 import { useState } from 'react'
 import { Link } from 'react-router'
 import Counter from '../../components/components.tsx'
-import style from './Template.module.css'
+//import style from './Template.module.css'
+import '../../App.css'
 
 //this is your css file where you can style your buttons and such
 //you can still use css parts from App.css, but also overwrite them

From d8ed8df9826b3de48a0b50fd7293577fb1a49eb3 Mon Sep 17 00:00:00 2001
From: Twirre Meulenbelt <43213592+TwirreM@users.noreply.github.com>
Date: Wed, 8 Oct 2025 15:56:34 +0200
Subject: [PATCH 4/4] feat: update robot interaction page

This page is now fancier, shows messages streaming from the Control Backend.

ref: N25B-164
---
 src/App.tsx                           |  2 +
 src/pages/Home/Home.tsx               |  1 +
 src/pages/Robot/Robot.tsx             | 94 +++++++++++++++++++++++++++
 src/pages/ServerComms/ServerComms.css |  0
 src/pages/ServerComms/ServerComms.tsx | 80 -----------------------
 5 files changed, 97 insertions(+), 80 deletions(-)
 create mode 100644 src/pages/Robot/Robot.tsx
 delete mode 100644 src/pages/ServerComms/ServerComms.css
 delete mode 100644 src/pages/ServerComms/ServerComms.tsx

diff --git a/src/App.tsx b/src/App.tsx
index acec25d..803b84c 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -2,6 +2,7 @@ import { Routes, Route, Link } from 'react-router'
 import './App.css'
 import TemplatePage from './pages/TemplatePage/Template.tsx'
 import Home from './pages/Home/Home.tsx'
+import Robot from './pages/Robot/Robot.tsx';
 
 function App(){
   return (
@@ -13,6 +14,7 @@ function App(){
         <Routes>
           <Route path="/" element={<Home />} />
           <Route path="/template" element={<TemplatePage />} />
+          <Route path="/robot" element={<Robot />} />
         </Routes>
       </main>
     </div>
diff --git a/src/pages/Home/Home.tsx b/src/pages/Home/Home.tsx
index 71f6b96..cb70de0 100644
--- a/src/pages/Home/Home.tsx
+++ b/src/pages/Home/Home.tsx
@@ -11,6 +11,7 @@ function Home() {
         </a>
       </div>
       <div className={styles.links}>
+        <Link to={"/robot"}>Robot Interaction →</Link>
         <Link to={"/template"}>Template →</Link>
       </div>
     </div>
diff --git a/src/pages/Robot/Robot.tsx b/src/pages/Robot/Robot.tsx
new file mode 100644
index 0000000..0038dd9
--- /dev/null
+++ b/src/pages/Robot/Robot.tsx
@@ -0,0 +1,94 @@
+import { useState, useEffect, useRef } from 'react'
+
+export default function Robot() {
+  const [message, setMessage] = useState('');
+
+  const [listening, setListening] = useState(false);
+  const [conversation, setConversation] = useState<{"role": "user" | "assistant", "content": string}[]>([])
+  const conversationRef = useRef<HTMLDivElement | null>(null);
+  const [conversationIndex, setConversationIndex] = useState(0);
+
+  const sendMessage = async () => {
+    try {
+      const response = await fetch("http://localhost:8000/message", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ message }),
+      });
+      const data = await response.json();
+      console.log(data);
+    } catch (error) {
+      console.error("Error sending message: ", error);
+    }
+  };
+
+  useEffect(() => {
+    const eventSource = new EventSource("http://localhost:8000/sse");
+
+    eventSource.onmessage = (event) => {
+      try {
+        const data = JSON.parse(event.data);
+        if ("voice_active" in data) setListening(data.voice_active);
+        if ("speech" in data) setConversation(conversation => [...conversation, {"role": "user", "content": data.speech}]);
+        if ("llm_response" in data) setConversation(conversation => [...conversation, {"role": "assistant", "content": data.llm_response}]);
+      } catch {
+        console.log("Unparsable SSE message:", event.data);
+      }
+    };
+
+    return () => {
+      eventSource.close();
+    };
+  }, [conversationIndex]);
+
+  useEffect(() => {
+    if (!conversationRef || !conversationRef.current) return;
+    conversationRef.current.scrollTop = conversationRef.current.scrollHeight;
+  }, [conversation]);
+
+  return (
+    <>
+      <h1>Robot interaction</h1>
+      <h2>Force robot speech</h2>
+      <div className={"flex-row gap-md justify-center"}>
+        <input
+          type="text"
+          value={message}
+          onChange={(e) => setMessage(e.target.value)}
+          onKeyDown={(e) => e.key === "Enter" && sendMessage().then(() => setMessage(""))}
+          placeholder="Enter a message"
+        />
+        <button onClick={sendMessage}>Speak</button>
+      </div>
+      <div className={"flex-col gap-lg align-center"}>
+        <h2>Conversation</h2>
+        <p>Listening {listening ? "🟢" : "🔴"}</p>
+        <div style={{ maxHeight: "200px", maxWidth: "600px", overflowY: "auto"}} ref={conversationRef}>
+          {conversation.map((item, i) => (
+            <p key={i}
+              style={{
+                backgroundColor: item["role"] == "user"
+                  ? "color-mix(in oklab, canvas, blue 20%)"
+                  : "color-mix(in oklab, canvas, gray 20%)",
+                whiteSpace: "pre-line",
+              }}
+              className={"round-md padding-md"}
+            >{item["content"]}</p>
+          ))}
+        </div>
+        <div className={"flex-row gap-md justify-center"}>
+          <button onClick={() => {
+            setConversationIndex((conversationIndex) => conversationIndex + 1)
+            setConversation([])
+          }}>Reset</button>
+          <button onClick={() => {
+            setConversationIndex((conversationIndex) => conversationIndex == -1 ? 0 : -1)
+            setConversation([])
+          }}>{conversationIndex == -1 ? "Start" : "Stop"}</button>
+        </div>
+      </div>
+    </>
+  );
+}
diff --git a/src/pages/ServerComms/ServerComms.css b/src/pages/ServerComms/ServerComms.css
deleted file mode 100644
index e69de29..0000000
diff --git a/src/pages/ServerComms/ServerComms.tsx b/src/pages/ServerComms/ServerComms.tsx
deleted file mode 100644
index c16ec81..0000000
--- a/src/pages/ServerComms/ServerComms.tsx
+++ /dev/null
@@ -1,80 +0,0 @@
-import { useState, useEffect } from 'react'
-import { Link } from 'react-router'
-//import Counter from '../../components/components.tsx'
-
-
-//this is your css file where you can style your buttons and such
-//you can still use css parts from App.css, but also overwrite them
-
-function ServerComms() {
-  const [message, setMessage] = useState('');
-  const [sseMessage, setSseMessage] = useState('');
-  const [spoken, setSpoken] = useState<string>("");
-
-  const sendMessage = async () => {
-    try {
-      const response = await fetch("http://localhost:8000/message", {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify({ message }),
-      });
-      const data = await response.json();
-      console.log(data);
-    } catch (error) {
-      console.error("Error sending message: ", error);
-    }
-  };
-
-  useEffect(() => {
-    const eventSource = new EventSource("http://localhost:8000/sse");
-
-    eventSource.onmessage = (event) => {
-      setSseMessage(event.data);
-
-      try {
-          const data = JSON.parse(event.data);
-          if (data.speech) setSpoken(data.speech);
-      } catch {}
-    };
-
-    return () => {
-      eventSource.close();
-    };
-  }, []);
-
-  return (
-    <div className="App">
-      <div>
-        <input
-          type="text"
-          value={message}
-          onChange={(e) => setMessage(e.target.value)}
-          onKeyDown={(e) => e.key === "Enter" && sendMessage().then(() => setMessage(""))}
-          placeholder="Enter a message"
-        />
-        <button onClick={sendMessage}>Send Message to Backend</button>
-      </div>
-      <div>
-        <h2>Message from Server (SSE):</h2>
-        <p>{sseMessage}</p>
-      </div>
-      <div>
-        <h2>Spoken text (SSE):</h2>
-        <p>{spoken}</p>
-      </div>
-      <div>
-        <Link to = {"/"}> {/* here you link to the homepage, in App.tsx you can link new pages */}
-          <button className= 'movePage left' onClick={() :void => {}}>
-            Page {'<'}-- Go Home
-          </button>
-        </Link>
-      </div>
-    </div>
-    
-    
-  );
-}
-
-export default ServerComms
\ No newline at end of file