Android API reference

public class ExtentosGlasses {
    public companion object {
        public fun create(config: ExtentosConfig = ExtentosConfig()): ExtentosGlasses
    }

    // Transport outcome — fixed for the life of the handle
    public val transportChosen: TransportChosen
    public val selectionSource: TransportSelectionSource

    // Sub-clients
    public val connection: ConnectionClient
    public val camera: CameraClient
    public val audio: AudioClient
    public val display: DisplayClient
    public val runtime: RuntimeClient
    public val toggles: ToggleClient
    public val voice: VoiceClient
    public val assistant: AssistantClient
    public val telemetry: TelemetryClient
    public val observability: ObservabilityClient
    public val device: DeviceIdentityClient
    public val connectionConfig: ConnectionConfigSource

    public val capabilities: GlassesCapabilities          // computed on every read
    public val usedCapabilities: Set<CapabilityKind>

    public suspend fun shutdown()
}

public data class ExtentosConfig(
    val appId: String? = null,
    val accountId: String? = null,
    val transport: TransportChoice = TransportChoice.Auto,
    val logLevel: LogLevel = LogLevel.WARN,
    val preferredStreamConfig: PreferredStreamConfig? = null,
    val debug: Boolean = false,
    val usedCapabilities: Set<CapabilityKind> = emptySet(),
    val telemetryConsent: Boolean = true,
    val dataSharingConsent: Boolean = true,
    val environment: ExtentosEnvironment = ExtentosEnvironment.DEVELOPMENT,
    val telemetryEndpoint: String? = null,
    val applicationContext: Context? = null,
    val activityProvider: (() -> Activity?)? = null,
    val hasBondedMetaDevice: (() -> Boolean)? = null,
)

public sealed interface TransportChoice {
    public data object Auto : TransportChoice
    public data object RealMeta : TransportChoice
    public data object SystemAudio : TransportChoice
    public sealed interface Simulated : TransportChoice {
        public data object Local : Simulated
        public data class Browser(val url: String? = null) : Simulated
    }
}

public enum class LogLevel { VERBOSE, DEBUG, INFO, WARN, ERROR }

public val state: StateFlow<GlassesState>
public val simulatorHint: StateFlow<SimulatorHint?>
public suspend fun connect(deviceId: DeviceId? = null): ExtentosResult<Unit, ConnectError>
public suspend fun disconnect()
public suspend fun reconnect(deviceId: DeviceId? = null): ExtentosResult<Unit, ConnectError>

public suspend fun capturePhoto(config: PhotoConfig = PhotoConfig()): ExtentosResult<Photo, CaptureError>
public suspend fun captureVideo(config: VideoConfig = VideoConfig()): ExtentosResult<VideoClip, CaptureError>
public suspend fun stopVideo()
public fun videoFrames(config: VideoFrameConfig = VideoFrameConfig()): Flow<VideoFrame>
public fun activeStreamInfo(): ActiveStreamInfo?          // a function; null when nothing is armed
public var preferredStreamConfig: PreferredStreamConfig?

try {
    glasses.camera.videoFrames().collect { render(it) }
} catch (e: CameraUnavailableException) {
    Log.e(TAG, "no camera on this transport: ${e.error}", e)
} catch (e: CameraStreamPausedException) {
    showHint("Tap the right temple of your glasses to resume the camera")
}

public suspend fun recordDiscrete(config: AudioRecordConfig = AudioRecordConfig()): ExtentosResult<AudioRecording, AudioError>
public fun audioChunks(config: AudioChunkConfig = AudioChunkConfig()): Flow<AudioChunk>
public fun transcriptions(config: TranscriptionConfig = TranscriptionConfig()): Flow<Transcript>
public suspend fun speak(text: String, config: SpeakConfig = SpeakConfig()): ExtentosResult<Unit, AudioError>
public suspend fun cancelSpeak()
public suspend fun earcon(sound: EarconSound, volume: Double = 1.0)
public suspend fun playSound(name: String, volume: Double = 1.0): ExtentosResult<Unit, AudioError>
public fun registerSound(name: String, pcm16: ByteArray, sampleRate: Int)
public fun soundNames(): List<String>

// Extension, not a member:
public fun AudioClient.startPushToTalk(
    scope: CoroutineScope,
    config: TranscriptionConfig = TranscriptionConfig(language = "en-US", partial = false),
): PushToTalkSession

public val hints: StateFlow<List<VoiceHint>>
public val stats: StateFlow<Map<String, VoiceHintStats>>

public fun onPhrase(
    phrase: String,
    label: String? = null,
    stops: List<String> = emptyList(),
    firesWhen: VoiceScope = VoiceScope.WhenDormant,
    handler: suspend () -> Unit,
): VoiceRegistration

public fun registerHint(phrase: String, label: String? = null, stops: List<String> = emptyList()): VoiceRegistration
public fun reportFired(id: String)

public enum class VoiceScope { Always, WhenDormant, WhenActive }
public interface VoiceRegistration { public fun cancel() }

public suspend fun start(provider: AssistantProvider, block: AssistantConfigBuilder.() -> Unit): AssistantSession
public fun createSession(config: AssistantConfig): AssistantSession
public suspend fun stop()
public val activeSession: AssistantSession?

// RuntimeClient
public val events: Flow<RuntimeEvent>
public suspend fun snapshotEvents(): List<RuntimeEvent>

// ToggleClient
public val state: StateFlow<Map<String, JSONValue>>
public fun put(key: String, value: JSONValue, source: ToggleSource = ToggleSource.UI)
public fun get(key: String): JSONValue?

// TelemetryClient
public fun setUserSegment(segment: String?)
public fun trackEvent(name: String, properties: Map<String, Any?> = emptyMap())
public val consent: Boolean

// ObservabilityClient — wrap your own AI calls so they land in the event log
public suspend fun <T> aiCall(label: String, metadata: Map<String, String> = emptyMap(), block: suspend () -> T): T

// DeviceIdentityClient
public val type: GlassesDeviceType
public val vendor: GlassesVendor

// ConnectionConfigSource
public fun cached(): JsonObject?
public suspend fun fetch(): JsonObject?

// DisplayClient — guard every call on isAvailable
public val isAvailable: Boolean
public suspend fun show(onBack: (() -> Unit)? = null, content: DisplayRootScope.() -> Unit)
public suspend fun clear()
public suspend fun prepareVideo(clip: VideoClip): Boolean
public suspend fun forgetHostedVideo(clip: VideoClip)
public suspend fun forgetHostedImage(photo: Photo)

public var instructions: String = ""
public var startActive: Boolean = false
public var endOnIntent: Boolean = true
public var greeting: Greeting = Greeting.Default
public var historyCap: Int = 100
public var historyCompaction: HistoryCompaction = HistoryCompaction.Auto
public var compactionModel: String? = null
public var persistentMemory: Boolean = false
public var memoryUserId: String? = null
public var memoryStore: MemoryStore? = null
public var wakeSoundEnabled: Boolean = true
public var localConductFloor: Boolean = true
public var fallbackGreeting: String? = null

public fun sleepAfterSilence(duration: Duration)   // kotlin.time.Duration; must be positive
public fun sleepOnPhrase(phrase: String)           // additive
public fun onWake(block: (suspend AssistantSession.() -> Unit)?)
public fun onSleep(block: (suspend AssistantSession.() -> Unit)?)

// tool() — three public overloads
public fun AssistantConfigBuilder.tool(
    name: String, description: String, blocking: Boolean = false,
    body: suspend () -> ToolResult,
)
public inline fun <reified Args> AssistantConfigBuilder.tool(
    name: String, description: String, blocking: Boolean = false,
    noinline body: suspend (Args) -> ToolResult,       // schema inferred from @Serializable
)
public fun AssistantConfigBuilder.tool(
    name: String, description: String, schema: JsonObject, blocking: Boolean = false,
    body: suspend (args: JsonObject) -> ToolResult,
)

public val config: AssistantConfig
public val state: StateFlow<AssistantState>

public suspend fun start()
public suspend fun wake()
public suspend fun sleep()
public suspend fun stop()
public suspend fun say(text: String)
public suspend fun greet(prompt: String? = null)
public suspend fun includeImage(uri: String, prompt: String? = null)
public suspend fun cancelSpeak()
public suspend fun setReasoningEffort(effort: ReasoningEffort)
public suspend fun setVoice(voice: String)
public suspend fun setModel(model: String)
public suspend fun updateInstructions(instructions: String)
public suspend fun sendVideoFrame(jpegBytes: ByteArray, mimeType: String = "image/jpeg")
public val modelSupportsVideoInput: Boolean

public fun conversationHistory(limit: Int = 100): List<Turn>
public fun clearHistory()
public fun appendHistory(turn: Turn)
public fun replaceHistory(turns: List<Turn>)

public enum class AssistantState {
    Idle, Dormant, Activating, Active, Reconnecting, Sleeping, Stopping, Stopped
}

public sealed class AssistantProvider {
    // One type for every model — the MODEL ID picks the vendor:
    // gpt-* OpenAI, grok-* xAI, gemini-* Google, local-* on-device.
    public data class Managed(
        val model: String? = null,          // "local-auto" or a local-* id runs on-device
        val voice: String? = null,
        val turnDetection: TurnDetection = TurnDetection.ServerVad(),
        val reasoningEffort: ReasoningEffort = ReasoningEffort.Low,
    ) : AssistantProvider()

    public data class Mock(
        val behavior: MockBehavior = MockBehavior.MatchToolDescriptions,
    ) : AssistantProvider()

    // Deprecated since 2.0.0, still fully functional — same shape, same
    // behaviour. Existing code compiles unchanged; migrate when convenient.
    @Deprecated("Renamed to AssistantProvider.Managed")
    public data class OpenAi(/* identical to Managed */) : AssistantProvider()
}

public enum class ReasoningEffort { Minimal, Low, Medium, High, Xhigh }

public sealed class TurnDetection {
    public data class ServerVad(
        val threshold: Double = 0.5,
        val prefixPaddingMs: Int = 300,
        val silenceDurationMs: Int = 500,
    ) : TurnDetection()
    public object SemanticVad : TurnDetection()
}

public sealed class ToolResult {
    public data class Ok(val output: String) : ToolResult()
    public data class Err(val message: String) : ToolResult()
}

public sealed class Greeting {
    public object Default : Greeting()
    public data class Custom(val directive: String) : Greeting()
    public object Off : Greeting()
}

// Unwrap a capture result inside a tool body, or bail with the SDK's own message.
// Constrained to CaptureError — it does NOT apply to AudioError or ConnectError.
public inline fun <T> ExtentosResult<T, CaptureError>.orToolError(
    onError: (ToolResult.Err) -> Nothing,
): T

public data class GlassesCapabilities(
    val camera: Boolean, val microphone: Boolean,
    val speaker: Boolean, val display: Boolean,
)

public sealed interface CapabilityKind {
    public data object Camera; public data object Microphone
    public data object Speaker; public data object Location
    public data object Notifications; public data object Display
    public data class Other(val name: String) : CapabilityKind
}

public enum class TransportChosen { REAL_META, BROWSER_SIM, LOCAL_SIM, PROJECTED_XR, SYSTEM_AUDIO }

public enum class TransportSelectionSource {
    BUILD_CONFIG, ENV_VAR, BONDED_DEVICES, FALLBACK_DEFAULT,
    EXPLICIT_CONFIG, PAIRING, VENDOR_REGISTRY
}

public enum class EarconSound { CONFIRMATION, ERROR, NOTIFICATION, START, STOP, COMPLETE }

public data class Photo(
    val uri: String?, val width: Int, val height: Int,
    val format: PhotoFormat, val exif: String?,
)
public data class VideoClip(
    val uri: String?, val durationMs: Long,
    val format: VideoFormat, val width: Int, val height: Int,
)

public data class PhotoConfig(
    val resolution: Resolution = Resolution.MEDIUM,
    val format: PhotoFormat = PhotoFormat.JPEG,
    val dedicatedCapture: Boolean = false,
)
public data class VideoConfig(
    val maxDurationSeconds: Int? = null, val includeAudio: Boolean = true,
    val format: VideoFormat = VideoFormat.MP4_HEVC,
    val resolution: Resolution = Resolution.MEDIUM, val frameRate: Int = 24,
)
public data class VideoFrameConfig(
    val frameRate: Int = 2, val resolution: Resolution = Resolution.LOW,
    val format: VideoFrameFormat = VideoFrameFormat.JPEG,
)
public data class AudioRecordConfig(
    val maxDurationSeconds: Int? = null, val silenceTimeoutSeconds: Double? = null,
    val quality: AudioQuality = AudioQuality.STANDARD,
)
public data class AudioChunkConfig(val chunkMillis: Int = 20, val sampleRate: Int = 16000)
public data class TranscriptionConfig(val language: String = "en-US", val partial: Boolean = true)
public data class SpeakConfig(
    val voice: String? = null, val rate: Double = 1.0, val pitch: Double = 1.0,
    val volume: Double = 1.0, val waitForCompletion: Boolean = true,
)

public sealed interface RuntimeEvent {
    public data class ToggleChanged(val key: String, val oldValue: JSONValue, val newValue: JSONValue, val source: ToggleSource) : RuntimeEvent
    public data class CoexistenceWarning(val blocked: String, val reason: String) : RuntimeEvent
    public data class Log(val level: LogLevel, val message: String, val payload: JSONValue? = null) : RuntimeEvent
    public data class UnrecognizedUtterance(val rawTranscript: String) : RuntimeEvent
    public data class Assistant(val event: AssistantEvent) : RuntimeEvent
}

public sealed interface AssistantEvent {
    public data class SessionStarted(val provider: String, val model: String?, val voice: String?)
    public data class SessionEnded(val reason: EndReason, val message: String? = null)
    public data class UserSpoke(val transcript: String)
    public data class AssistantSpoke(val transcript: String)
    public data class ToolCalled(val name: String, val args: JsonObject, val callId: String)
    public data class ToolResultEvent(val callId: String, val name: String, val output: String, val isError: Boolean, val durationMs: Long)
    public data class Reconnected(val reason: ReconnectReason, val downtimeMs: Long)
    public data class Error(val kind: String, val message: String)
    public data class LocalModelRequired(val modelId: String, val reason: LocalModelRequirement.Reason, val spokenMessage: String)
    public data class AutoModelResolved(val servedModelId: String, val isLocal: Boolean, val cloudReason: CloudFallbackReason?, val downloadTarget: String?)
    public object WentDormant
}

public sealed class GlassesState {
    public object NotRegistered
    public object Registered
    public data class DeviceDiscovered(val deviceId: String)
    public data class Connecting(val deviceId: String)
    public data class Active(val state: ActiveState)
    public data class Disconnected(val cause: DisconnectCause)
}

public open class GlassesForegroundService : Service() {
    public companion object {
        public const val CHANNEL_ID: String = "extentos_capture"

        @JvmStatic @JvmOverloads
        public fun start(context: Context, title: String? = null,
                         text: String? = null, smallIconResId: Int = 0)

        @JvmStatic @JvmOverloads
        public fun start(context: Context, serviceClass: Class<out GlassesForegroundService>,
                         title: String? = null, text: String? = null, smallIconResId: Int = 0)

        @JvmStatic @JvmOverloads
        public fun stop(context: Context,
                        serviceClass: Class<out GlassesForegroundService> = GlassesForegroundService::class.java)
    }
}

// com.extentos.glasses.local  (com.extentos:glasses-local)
public object ExtentosLocalTier {
    public fun register(context: Context)                    // before ExtentosGlasses.create(...)
    public fun models(context: Context): List<LocalModelInfo>
    public suspend fun download(
        context: Context, modelId: String,
        onProgress: (bytesDownloaded: Long, totalBytes: Long) -> Unit = { _, _ -> },
    ): Boolean
    public fun delete(context: Context, modelId: String): Boolean
    public fun autoChoice(context: Context): AutoChoice
    public fun deviceFit(context: Context, modelId: String): DeviceFit

    public data class LocalModelInfo(val id: String, val displayName: String, val totalBytes: Long, val isInstalled: Boolean)
    public data class AutoChoice(val modelId: String?, val isLocal: Boolean, val cloudReason: String?, val downloadTarget: String?)
    public data class DeviceFit(val requiredMb: Long, val availableMb: Long, val fits: Boolean)
}

// com.extentos.glasses.localvoice  (com.extentos:glasses-local-voice)
public object ExtentosLocalVoice {
    public fun register(context: Context)            // registers BOTH the voice and the recogniser
    public fun modelDirectory(context: Context): File
    public fun sttModelDirectory(context: Context): File
}

// On-device voice (~330 MB) and on-device transcription (~73 MB). Identical
// contract: size up front, byte progress, resumable, deletable, and never
// downloaded unless the app asks. Until installed, the system voice /
// recogniser serves.
public object KokoroVoiceModel {
    public val totalBytes: Long
    public fun isInstalled(context: Context): Boolean
    public fun delete(context: Context): Boolean
    public suspend fun download(
        context: Context,
        onProgress: (bytesDownloaded: Long, totalBytes: Long) -> Unit = { _, _ -> },
    )
}

public object ExtentosSttModel {
    public val totalBytes: Long
    public fun isInstalled(context: Context): Boolean
    public fun delete(context: Context): Boolean
    public suspend fun download(
        context: Context,
        onProgress: (bytesDownloaded: Long, totalBytes: Long) -> Unit = { _, _ -> },
    )
}