Build a voice assistant

class MyApp : Application() {
    lateinit var glasses: ExtentosGlasses

    override fun onCreate() {
        super.onCreate()
        glasses = ExtentosGlasses.create(ExtentosConfig(applicationContext = this))
        // No OpenAI key anywhere — the assistant runs on the Extentos managed
        // gateway by default. To use your own key, add it in the dashboard
        // Credentials section; the gateway swaps it in server-side.
        RunAssistantHandler(glasses, routeTracker, library).start()
    }
}

import com.extentos.glasses.core.ExtentosGlasses
import com.extentos.glasses.core.RuntimeEvent
import com.extentos.glasses.core.VideoClip
import com.extentos.glasses.core.VideoConfig
import com.extentos.glasses.core.valueOrNull
import com.extentos.glasses.core.assistant.AssistantEvent
import com.extentos.glasses.core.assistant.AssistantProvider
import com.extentos.glasses.core.assistant.AssistantSession
import com.extentos.glasses.core.assistant.Greeting
import com.extentos.glasses.core.assistant.ToolResult
import com.extentos.glasses.core.assistant.tool
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlin.time.Duration.Companion.seconds

class RunAssistantHandler(
    private val glasses: ExtentosGlasses,
    private val routeTracker: RouteTracker,   // your app state
    private val library: ClipLibrary,         // your app state
    private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO),
) {
    private var session: AssistantSession? = null
    private var activeVideo: Deferred<*>? = null

    fun start() {
        scope.launch {
            session = glasses.assistant.start(provider = AssistantProvider.OpenAi()) {
                instructions = """
                    You are a running companion. Help with route stats and capture
                    moments. Speak briefly — they're running. Don't narrate what
                    you're doing; just do it and confirm. When the user clearly
                    wants to stop talking, call end_conversation.
                """.trimIndent()

                // The SDK speaks this greeting automatically on every wake,
                // generated out-of-band from memory (never continues the prior chat).
                greeting = Greeting.Custom(
                    "Greet the runner warmly in one short sentence and ask how you can help."
                )
                // Deterministic backup sleep after 30s of user silence.
                sleepAfterSilence(30.seconds)

                // Read tools — instant data the model reads aloud.
                tool("get_route_remaining", "How much of the planned route is left, in km.") {
                    ToolResult.Ok("${'$'}{routeTracker.kmRemaining()} km remaining")
                }
                tool("get_average_pace", "The runner's current average pace in minutes per km.") {
                    ToolResult.Ok("${'$'}{routeTracker.avgPaceMinKm()} min per km")
                }

                // Action tools — side effects on your own state. The model
                // manages the take/stop pair from conversational context.
                tool("take_video", "Start recording a video clip of the runner's view.") {
                    if (activeVideo?.isActive == true) {
                        return@tool ToolResult.Err("a recording is already in progress")
                    }
                    activeVideo = scope.async {
                        glasses.camera.captureVideo(VideoConfig(maxDurationSeconds = 30))
                    }
                    ToolResult.Ok("recording started")
                }
                tool("stop_video", "Stop the current video recording.") {
                    val capture = activeVideo ?: return@tool ToolResult.Err("nothing was recording")
                    activeVideo = null
                    glasses.camera.stopVideo()                 // resumes the await naturally
                    @Suppress("UNCHECKED_CAST")
                    val clip = (capture as Deferred<com.extentos.glasses.core.ExtentosResult<VideoClip, *>>)
                        .await().valueOrNull() ?: return@tool ToolResult.Err("video capture failed")
                    library.add(clip)
                    ToolResult.Ok("video saved")
                }
            }

            // Wake mechanism — the same voice-trigger system as everywhere.
            // Defaults to VoiceScope.WhenDormant so it won't double-fire mid-chat.
            glasses.voice.onPhrase("hey coach") { session?.wake() }
        }
    }
}

tool(
    "describe_scene",
    "Describe what the runner is looking at without saving the photo. " +
        "Call for 'what do you see' / 'describe this' / 'tell me about this'.",
) {
    val photo = glasses.camera.capturePhoto().valueOrNull()
        ?: return@tool ToolResult.Err("camera failed")
    val uri = photo.uri ?: return@tool ToolResult.Err("photo had no uri")
    glasses.assistant.activeSession?.includeImage(uri)
    ToolResult.Ok("looking")
}

glasses.runtime.events
    .filterIsInstance<RuntimeEvent.Assistant>()
    .onEach { evt ->
        (evt.event as? AssistantEvent.UserSpoke)?.let { notes.append(it.transcript) }
    }
    .launchIn(scope)

// Wake exactly like a real user saying the phrase:
await injectTranscript({ sessionId, text: "hey coach" });
// ... wait for assistant.session_started in getEventLog(filter: "ai"), then:
const inj = await injectAssistantUtterance({ sessionId, text: "how much further?" });
await assertToolCalled({ sessionId, name: "get_route_remaining", sinceCursor: inj.watchCursor });