Build a voice assistant

class MyApp : Application() {
    lateinit var glasses: ExtentosGlasses

    override fun onCreate() {
        super.onCreate()
        // Cheap: resolves the transport, wires the clients. No mic yet.
        // No OpenAI key anywhere — the assistant runs on the Extentos managed
        // gateway (there is no option to run it on your own OpenAI account).
        glasses = ExtentosGlasses.create(ExtentosConfig(applicationContext = this))
    }
}

class MainActivity : ComponentActivity() {

    private val micPermission = registerForActivityResult(
        ActivityResultContracts.RequestPermission(),
    ) { granted -> if (granted) onMicGranted() }

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
            onMicGranted()
        } else {
            micPermission.launch(Manifest.permission.RECORD_AUDIO)
        }
    }

    private fun onMicGranted() {
        // Only now: the foreground service (so the mic survives backgrounding)
        // and the assistant handler.
        GlassesForegroundService.start(this)
        // Your handler — step 2 defines it. routeTracker / library are your own app state.
        RunAssistantHandler((application as MyApp).glasses, routeTracker, library).start()
    }
}

class MyApp : Application() {
    lateinit var glasses: ExtentosGlasses
    private val topActivity = java.util.concurrent.atomic.AtomicReference<Activity?>()

    override fun onCreate() {
        super.onCreate()
        registerActivityLifecycleCallbacks(object : ActivityLifecycleCallbacks {
            override fun onActivityResumed(activity: Activity) { topActivity.set(activity) }
            override fun onActivityPaused(activity: Activity) {
                if (topActivity.get() === activity) topActivity.set(null)
            }
            // remaining callbacks: no-op
            override fun onActivityCreated(a: Activity, b: Bundle?) {}
            override fun onActivityStarted(a: Activity) {}
            override fun onActivityStopped(a: Activity) {}
            override fun onActivitySaveInstanceState(a: Activity, b: Bundle) {}
            override fun onActivityDestroyed(a: Activity) {}
        })
    }

    /** Called from the Activity once the permissions below are granted. */
    fun initGlasses() {
        if (::glasses.isInitialized) return
        glasses = ExtentosGlasses.create(
            ExtentosConfig(
                applicationContext = this,
                activityProvider = { topActivity.get() },
                environment = if (BuildConfig.DEBUG) ExtentosEnvironment.DEVELOPMENT
                              else ExtentosEnvironment.PRODUCTION,
            )
        )
    }
}

class MainActivity : ComponentActivity() {

    private val permissions = registerForActivityResult(
        ActivityResultContracts.RequestMultiplePermissions(),
    ) { result ->
        // Create only after the answer, whatever it was — a denied BLUETOOTH_CONNECT
        // still resolves correctly (to the audio baseline), it just isn't a surprise.
        val app = application as MyApp
        app.initGlasses()
        Log.i("Extentos", "transport=${app.glasses.transportChosen} camera=${app.glasses.capabilities.camera}")
        if (result[Manifest.permission.RECORD_AUDIO] == true) onMicGranted()
    }

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        permissions.launch(
            arrayOf(
                Manifest.permission.RECORD_AUDIO,
                Manifest.permission.CAMERA,
                Manifest.permission.BLUETOOTH_CONNECT,
            )
        )
    }

    private fun onMicGranted() {
        GlassesForegroundService.start(this)
        // Your handler — step 2 defines it.
        RunAssistantHandler((application as MyApp).glasses, routeTracker, library).start()
    }
}

Log.i("Extentos", "transport=${glasses.transportChosen} camera=${glasses.capabilities.camera}")

import com.extentos.glasses.core.ExtentosGlasses
import com.extentos.glasses.core.RuntimeEvent
import com.extentos.glasses.core.VideoClip
import com.extentos.glasses.core.VideoConfig
import com.extentos.glasses.core.valueOrNull
import com.extentos.glasses.core.assistant.AssistantEvent
import com.extentos.glasses.core.assistant.AssistantError
import com.extentos.glasses.core.assistant.AssistantException
import com.extentos.glasses.core.assistant.AssistantProvider
import com.extentos.glasses.core.assistant.AssistantSession
import com.extentos.glasses.core.assistant.Greeting
import com.extentos.glasses.core.assistant.ToolResult
import com.extentos.glasses.core.assistant.tool
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlin.time.Duration.Companion.seconds

class RunAssistantHandler(
    private val glasses: ExtentosGlasses,
    private val routeTracker: RouteTracker,   // your app state
    private val library: ClipLibrary,         // your app state
    private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO),
) {
    private var session: AssistantSession? = null
    private var activeVideo: Deferred<*>? = null

    fun start() {
        scope.launch {
            session = glasses.assistant.start(provider = AssistantProvider.Managed()) {
                instructions = """
                    You are a running companion. Help with route stats and capture
                    moments. Speak briefly — they're running. Don't narrate what
                    you're doing; just do it and confirm. When the user clearly
                    wants to stop talking, call end_conversation.
                """.trimIndent()

                // The SDK speaks this greeting automatically on every wake,
                // generated out-of-band from memory (never continues the prior chat).
                greeting = Greeting.Custom(
                    "Greet the runner warmly in one short sentence and ask how you can help."
                )
                // Deterministic backup sleep after 30s of user silence.
                sleepAfterSilence(30.seconds)

                // Read tools — instant data the model reads aloud.
                tool("get_route_remaining", "How much of the planned route is left, in km.") {
                    ToolResult.Ok("${routeTracker.kmRemaining()} km remaining")
                }
                tool("get_average_pace", "The runner's current average pace in minutes per km.") {
                    ToolResult.Ok("${routeTracker.avgPaceMinKm()} min per km")
                }

                // Action tools — side effects on your own state. The model
                // manages the take/stop pair from conversational context.
                tool("take_video", "Start recording a video clip of the runner's view.") {
                    if (activeVideo?.isActive == true) {
                        return@tool ToolResult.Err("a recording is already in progress")
                    }
                    activeVideo = scope.async {
                        glasses.camera.captureVideo(VideoConfig(maxDurationSeconds = 30))
                    }
                    ToolResult.Ok("recording started")
                }
                tool("stop_video", "Stop the current video recording.") {
                    val capture = activeVideo ?: return@tool ToolResult.Err("nothing was recording")
                    activeVideo = null
                    glasses.camera.stopVideo()                 // resumes the await naturally
                    @Suppress("UNCHECKED_CAST")
                    val clip = (capture as Deferred<com.extentos.glasses.core.ExtentosResult<VideoClip, *>>)
                        .await().valueOrNull() ?: return@tool ToolResult.Err("video capture failed")
                    library.add(clip)
                    ToolResult.Ok("video saved")
                }
            }

            // Wake mechanism — the same voice-trigger system as everywhere.
            // Defaults to VoiceScope.WhenDormant so it won't double-fire mid-chat.
            glasses.voice.onPhrase("hey coach") { session?.wake() }
        }
    }
}

import com.extentos.glasses.core.assistant.orToolError

tool(
    "describe_scene",
    "Describe what the runner is looking at without saving the photo. " +
        "Call for 'what do you see' / 'describe this' / 'tell me about this'.",
) {
    val photo = glasses.camera.capturePhoto().orToolError { return@tool it }
    val uri = photo.uri ?: return@tool ToolResult.Err("photo had no uri")
    glasses.assistant.activeSession?.includeImage(uri)
    ToolResult.Ok("looking")
}

glasses.runtime.events
    .filterIsInstance<RuntimeEvent.Assistant>()
    .onEach { evt ->
        (evt.event as? AssistantEvent.UserSpoke)?.let { notes.append(it.transcript) }
    }
    .launchIn(scope)

// Wake exactly like a real user saying the phrase:
await injectTranscript({ sessionId, text: "hey coach" });
// ... wait for assistant.session_started in getEventLog(filter: "voice"), then:
const inj = await injectAssistantUtterance({ sessionId, text: "how much further?" });
await assertToolCalled({ sessionId, name: "get_route_remaining", sinceCursor: inj.watchCursor });