Plan Agent targets before launching Genies

Teach the Agent runtime and UI to resolve target packages from installed apps and start one child Genie session per selected package, with an optional package override for debugging.

Co-authored-by: Codex <noreply@openai.com>

android/app/src/main/java/com/openai/codexd/AgentInstalledAppCatalog.kt

@@ -0,0 +1,28 @@
+package com.openai.codexd
+
+import android.content.Context
+import android.content.Intent
+
+data class InstalledLaunchableApp(
+    val packageName: String,
+    val label: String,
+)
+
+object AgentInstalledAppCatalog {
+    fun listLaunchableApps(context: Context): List<InstalledLaunchableApp> {
+        val packageManager = context.packageManager
+        return packageManager.queryIntentActivities(
+            Intent(Intent.ACTION_MAIN).addCategory(Intent.CATEGORY_LAUNCHER),
+            0,
+        )
+            .map { resolveInfo ->
+                val packageName = resolveInfo.activityInfo.packageName
+                InstalledLaunchableApp(
+                    packageName = packageName,
+                    label = resolveInfo.loadLabel(packageManager)?.toString().orEmpty().ifBlank { packageName },
+                )
+            }
+            .distinctBy(InstalledLaunchableApp::packageName)
+            .sortedWith(compareBy(InstalledLaunchableApp::label, InstalledLaunchableApp::packageName))
+    }
+}

android/app/src/main/java/com/openai/codexd/AgentSessionController.kt

@@ -81,8 +81,7 @@ class AgentSessionController(context: Context) {
     }
 
     fun startDirectSession(
-        targetPackage: String,
-        prompt: String,
+        plan: AgentDelegationPlan,
         allowDetachedMode: Boolean,
     ): SessionStartResult {
         val manager = requireAgentManager()
@@ -93,23 +92,29 @@ class AgentSessionController(context: Context) {
         try {
             manager.publishTrace(
                 parentSession.sessionId,
-                "Starting Codex direct session for $targetPackage.",
-            )
-            val childSession = manager.createChildSession(parentSession.sessionId, targetPackage)
-            childSessionIds += childSession.sessionId
-            manager.publishTrace(
-                parentSession.sessionId,
-                "Created child session ${childSession.sessionId} for $targetPackage.",
-            )
-            manager.startGenieSession(
-                childSession.sessionId,
-                geniePackage,
-                prompt,
-                allowDetachedMode,
+                "Starting Codex direct session for objective: ${plan.originalObjective}",
             )
+            plan.rationale?.let { rationale ->
+                manager.publishTrace(parentSession.sessionId, "Planning rationale: $rationale")
+            }
+            plan.targets.forEach { target ->
+                val childSession = manager.createChildSession(parentSession.sessionId, target.packageName)
+                childSessionIds += childSession.sessionId
+                manager.publishTrace(
+                    parentSession.sessionId,
+                    "Created child session ${childSession.sessionId} for ${target.packageName}.",
+                )
+                manager.startGenieSession(
+                    childSession.sessionId,
+                    geniePackage,
+                    target.objective,
+                    allowDetachedMode,
+                )
+            }
             return SessionStartResult(
                 parentSessionId = parentSession.sessionId,
-                childSessionId = childSession.sessionId,
+                childSessionIds = childSessionIds,
+                plannedTargets = plan.targets.map(AgentDelegationTarget::packageName),
                 geniePackage = geniePackage,
             )
         } catch (err: RuntimeException) {
@@ -315,7 +320,8 @@ data class AgentSessionDetails(
 
 data class SessionStartResult(
     val parentSessionId: String,
-    val childSessionId: String,
+    val childSessionIds: List<String>,
+    val plannedTargets: List<String>,
     val geniePackage: String,
 )
 

android/app/src/main/java/com/openai/codexd/AgentTaskPlanner.kt

@@ -0,0 +1,143 @@
+package com.openai.codexd
+
+import android.content.Context
+import java.io.IOException
+import org.json.JSONArray
+import org.json.JSONObject
+
+data class AgentDelegationTarget(
+    val packageName: String,
+    val objective: String,
+)
+
+data class AgentDelegationPlan(
+    val originalObjective: String,
+    val targets: List<AgentDelegationTarget>,
+    val rationale: String?,
+    val usedOverride: Boolean,
+) {
+    val primaryTargetPackage: String
+        get() = targets.first().packageName
+}
+
+object AgentTaskPlanner {
+    private const val MAX_LAUNCHABLE_APPS = 80
+    private val PLANNER_INSTRUCTIONS =
+        """
+        You are Codex acting as the Android Agent planner.
+        The user interacts only with the Agent. Decide which installed Android packages should receive delegated Genie sessions.
+        Choose the fewest packages needed to complete the request.
+        Return exactly one JSON object with this shape:
+        {"targets":[{"packageName":"com.example.app","objective":"free-form delegated objective"}],"reason":"short explanation"}
+        Rules:
+        - Use only package names from the provided installed-app list.
+        - `targets` must be non-empty.
+        - Each delegated `objective` should be written for the child Genie, not the user.
+        - Do not include markdown or code fences.
+        """.trimIndent()
+
+    fun plan(
+        context: Context,
+        userObjective: String,
+        targetPackageOverride: String?,
+    ): AgentDelegationPlan {
+        if (!targetPackageOverride.isNullOrBlank()) {
+            return AgentDelegationPlan(
+                originalObjective = userObjective,
+                targets = listOf(
+                    AgentDelegationTarget(
+                        packageName = targetPackageOverride,
+                        objective = userObjective,
+                    ),
+                ),
+                rationale = "Using explicit target package override.",
+                usedOverride = true,
+            )
+        }
+        val runtimeStatus = CodexdLocalClient.waitForRuntimeStatus(context)
+        if (!runtimeStatus.authenticated) {
+            throw IOException("codexd is not authenticated")
+        }
+        val model = runtimeStatus.effectiveModel ?: throw IOException("codexd effective model unavailable")
+        val launchableApps = AgentInstalledAppCatalog.listLaunchableApps(context)
+            .take(MAX_LAUNCHABLE_APPS)
+        if (launchableApps.isEmpty()) {
+            throw IOException("No launchable apps available for planning")
+        }
+        val planText = CodexResponsesClient.requestText(
+            context = context,
+            model = model,
+            instructions = PLANNER_INSTRUCTIONS,
+            prompt = buildPlannerPrompt(userObjective, launchableApps),
+        )
+        return parsePlanResponse(
+            responseText = planText,
+            userObjective = userObjective,
+            allowedPackageNames = launchableApps.mapTo(linkedSetOf(), InstalledLaunchableApp::packageName),
+        )
+    }
+
+    internal fun parsePlanResponse(
+        responseText: String,
+        userObjective: String,
+        allowedPackageNames: Set<String>,
+    ): AgentDelegationPlan {
+        val responseJson = extractJsonObject(responseText)
+        val targetsJson = responseJson.optJSONArray("targets")
+            ?: throw IOException("Planner response missing targets")
+        val targets = buildList {
+            for (index in 0 until targetsJson.length()) {
+                val target = targetsJson.optJSONObject(index) ?: continue
+                val packageName = target.optString("packageName").trim()
+                if (packageName.isEmpty() || !allowedPackageNames.contains(packageName)) {
+                    continue
+                }
+                val objective = target.optString("objective").trim().ifEmpty { userObjective }
+                add(
+                    AgentDelegationTarget(
+                        packageName = packageName,
+                        objective = objective,
+                    ),
+                )
+            }
+        }.distinctBy(AgentDelegationTarget::packageName)
+        if (targets.isEmpty()) {
+            throw IOException("Planner response did not select an installed package")
+        }
+        return AgentDelegationPlan(
+            originalObjective = userObjective,
+            targets = targets,
+            rationale = responseJson.optString("reason").ifBlank { null },
+            usedOverride = false,
+        )
+    }
+
+    private fun buildPlannerPrompt(
+        userObjective: String,
+        launchableApps: List<InstalledLaunchableApp>,
+    ): String {
+        val appList = launchableApps.joinToString(separator = "\n") { app ->
+            "- ${app.label} (${app.packageName})"
+        }
+        return """
+            User objective:
+            $userObjective
+
+            Installed launchable apps:
+            $appList
+        """.trimIndent()
+    }
+
+    private fun extractJsonObject(responseText: String): JSONObject {
+        val start = responseText.indexOf('{')
+        val end = responseText.lastIndexOf('}')
+        if (start == -1 || end == -1 || end <= start) {
+            throw IOException("Planner response did not contain JSON")
+        }
+        return try {
+            JSONObject(responseText.substring(start, end + 1))
+        } catch (err: Exception) {
+            throw IOException("Planner response was not valid JSON: ${err.message}", err)
+        }
+    }
+}

android/app/src/main/java/com/openai/codexd/CodexAgentService.kt

@@ -6,8 +6,6 @@ import android.app.agent.AgentSessionEvent
 import android.app.agent.AgentSessionInfo
 import android.os.Process
 import android.util.Log
-import org.json.JSONArray
-import org.json.JSONObject
 import java.io.IOException
 import kotlin.concurrent.thread
 
@@ -135,32 +133,12 @@ class CodexAgentService : AgentService() {
             throw IOException("codexd is not authenticated")
         }
         val model = runtimeStatus.effectiveModel ?: throw IOException("codexd effective model unavailable")
-        val requestBody = JSONObject()
-            .put("model", model)
-            .put("store", false)
-            .put("stream", true)
-            .put("instructions", AUTO_ANSWER_INSTRUCTIONS)
-            .put(
-                "input",
-                JSONArray().put(
-                    JSONObject()
-                        .put("role", "user")
-                        .put(
-                            "content",
-                            JSONArray().put(
-                                JSONObject()
-                                    .put("type", "input_text")
-                                    .put("text", buildAutoAnswerPrompt(session, question, events)),
-                            ),
-                        ),
-                ),
-            )
-            .toString()
-        val response = CodexdLocalClient.waitForResponse(this, "POST", "/v1/responses", requestBody)
-        if (response.statusCode != 200) {
-            throw IOException("HTTP ${response.statusCode}: ${response.body}")
-        }
-        return parseResponsesOutputText(response.body)
+        return CodexResponsesClient.requestText(
+            context = this,
+            model = model,
+            instructions = AUTO_ANSWER_INSTRUCTIONS,
+            prompt = buildAutoAnswerPrompt(session, question, events),
+        )
     }
 
     private fun buildAutoAnswerPrompt(
@@ -189,83 +167,6 @@ class CodexAgentService : AgentService() {
         }
         return context.takeLast(MAX_AUTO_ANSWER_CONTEXT_CHARS)
     }
-
-    private fun parseResponsesOutputText(body: String): String {
-        val trimmedBody = body.trim()
-        if (trimmedBody.startsWith("event:") || trimmedBody.startsWith("data:")) {
-            return parseResponsesStreamOutputText(trimmedBody)
-        }
-        val data = JSONObject(trimmedBody)
-        val directOutput = data.optString("output_text")
-        if (directOutput.isNotBlank()) {
-            return directOutput
-        }
-        val output = data.optJSONArray("output")
-            ?: throw IOException("Responses payload missing output")
-        val combined = buildString {
-            for (outputIndex in 0 until output.length()) {
-                val item = output.optJSONObject(outputIndex) ?: continue
-                val content = item.optJSONArray("content") ?: continue
-                for (contentIndex in 0 until content.length()) {
-                    val part = content.optJSONObject(contentIndex) ?: continue
-                    if (part.optString("type") == "output_text") {
-                        append(part.optString("text"))
-                    }
-                }
-            }
-        }
-        if (combined.isBlank()) {
-            throw IOException("Responses payload missing output_text content")
-        }
-        return combined
-    }
-
-    private fun parseResponsesStreamOutputText(body: String): String {
-        val deltaText = StringBuilder()
-        val completedItems = mutableListOf<String>()
-        body.split("\n\n").forEach { rawEvent ->
-            val lines = rawEvent.lineSequence().map(String::trimEnd).toList()
-            if (lines.isEmpty()) {
-                return@forEach
-            }
-            val dataPayload = lines
-                .filter { it.startsWith("data:") }
-                .joinToString("\n") { it.removePrefix("data:").trimStart() }
-                .trim()
-            if (dataPayload.isEmpty() || dataPayload == "[DONE]") {
-                return@forEach
-            }
-            val event = JSONObject(dataPayload)
-            when (event.optString("type")) {
-                "response.output_text.delta" -> deltaText.append(event.optString("delta"))
-                "response.output_item.done" -> {
-                    val item = event.optJSONObject("item") ?: return@forEach
-                    val content = item.optJSONArray("content") ?: return@forEach
-                    val text = buildString {
-                        for (index in 0 until content.length()) {
-                            val part = content.optJSONObject(index) ?: continue
-                            if (part.optString("type") == "output_text") {
-                                append(part.optString("text"))
-                            }
-                        }
-                    }
-                    if (text.isNotBlank()) {
-                        completedItems += text
-                    }
-                }
-                "response.failed" -> throw IOException(event.toString())
-            }
-        }
-        if (deltaText.isNotBlank()) {
-            return deltaText.toString()
-        }
-        val completedText = completedItems.joinToString("")
-        if (completedText.isNotBlank()) {
-            return completedText
-        }
-        throw IOException("Responses stream missing output_text content")
-    }
-
     private fun findVisibleQuestion(events: List<AgentSessionEvent>): String? {
         return events.lastOrNull { event ->
             event.type == AgentSessionEvent.TYPE_QUESTION &&

android/app/src/main/java/com/openai/codexd/CodexResponsesClient.kt

@@ -0,0 +1,136 @@
+package com.openai.codexd
+
+import java.io.IOException
+import org.json.JSONArray
+import org.json.JSONObject
+
+object CodexResponsesClient {
+    fun requestText(
+        context: android.content.Context,
+        model: String,
+        instructions: String,
+        prompt: String,
+    ): String {
+        val requestBody = buildRequest(
+            model = model,
+            instructions = instructions,
+            prompt = prompt,
+        )
+        val response = CodexdLocalClient.waitForResponse(
+            context = context,
+            method = "POST",
+            path = "/v1/responses",
+            body = requestBody.toString(),
+        )
+        if (response.statusCode != 200) {
+            throw IOException("HTTP ${response.statusCode}: ${response.body}")
+        }
+        return parseResponsesOutputText(response.body)
+    }
+
+    fun buildRequest(
+        model: String,
+        instructions: String,
+        prompt: String,
+    ): JSONObject {
+        return JSONObject()
+            .put("model", model)
+            .put("store", false)
+            .put("stream", true)
+            .put("instructions", instructions)
+            .put(
+                "input",
+                JSONArray().put(
+                    JSONObject()
+                        .put("role", "user")
+                        .put(
+                            "content",
+                            JSONArray().put(
+                                JSONObject()
+                                    .put("type", "input_text")
+                                    .put("text", prompt),
+                            ),
+                        ),
+                ),
+            )
+    }
+
+    fun parseResponsesOutputText(body: String): String {
+        val trimmedBody = body.trim()
+        if (trimmedBody.startsWith("event:") || trimmedBody.startsWith("data:")) {
+            return parseResponsesStreamOutputText(trimmedBody)
+        }
+        return parseResponsesJsonOutputText(JSONObject(trimmedBody))
+    }
+
+    private fun parseResponsesJsonOutputText(data: JSONObject): String {
+        val directOutput = data.optString("output_text")
+        if (directOutput.isNotBlank()) {
+            return directOutput
+        }
+        val output = data.optJSONArray("output")
+            ?: throw IOException("Responses payload missing output")
+        val combined = buildString {
+            for (outputIndex in 0 until output.length()) {
+                val item = output.optJSONObject(outputIndex) ?: continue
+                val content = item.optJSONArray("content") ?: continue
+                for (contentIndex in 0 until content.length()) {
+                    val part = content.optJSONObject(contentIndex) ?: continue
+                    if (part.optString("type") == "output_text") {
+                        append(part.optString("text"))
+                    }
+                }
+            }
+        }
+        if (combined.isBlank()) {
+            throw IOException("Responses payload missing output_text content")
+        }
+        return combined
+    }
+
+    private fun parseResponsesStreamOutputText(body: String): String {
+        val deltaText = StringBuilder()
+        val completedItems = mutableListOf<String>()
+        body.split("\n\n").forEach { rawEvent ->
+            val lines = rawEvent.lineSequence().map(String::trimEnd).toList()
+            if (lines.isEmpty()) {
+                return@forEach
+            }
+            val dataPayload = lines
+                .filter { it.startsWith("data:") }
+                .joinToString("\n") { it.removePrefix("data:").trimStart() }
+                .trim()
+            if (dataPayload.isEmpty() || dataPayload == "[DONE]") {
+                return@forEach
+            }
+            val event = JSONObject(dataPayload)
+            when (event.optString("type")) {
+                "response.output_text.delta" -> deltaText.append(event.optString("delta"))
+                "response.output_item.done" -> {
+                    val item = event.optJSONObject("item") ?: return@forEach
+                    val content = item.optJSONArray("content") ?: return@forEach
+                    val text = buildString {
+                        for (index in 0 until content.length()) {
+                            val part = content.optJSONObject(index) ?: continue
+                            if (part.optString("type") == "output_text") {
+                                append(part.optString("text"))
+                            }
+                        }
+                    }
+                    if (text.isNotBlank()) {
+                        completedItems += text
+                    }
+                }
+                "response.failed" -> throw IOException(event.toString())
+            }
+        }
+        if (deltaText.isNotBlank()) {
+            return deltaText.toString()
+        }
+        val completedText = completedItems.joinToString("")
+        if (completedText.isNotBlank()) {
+            return completedText
+        }
+        throw IOException("Responses stream missing output_text content")
+    }
+}

android/app/src/main/java/com/openai/codexd/MainActivity.kt

@@ -171,12 +171,8 @@ class MainActivity : Activity() {
     }
 
     fun startDirectAgentSession(@Suppress("UNUSED_PARAMETER") view: View) {
-        val targetPackage = findViewById<EditText>(R.id.agent_target_package).text.toString().trim()
+        val targetPackageOverride = findViewById<EditText>(R.id.agent_target_package).text.toString().trim()
         val prompt = findViewById<EditText>(R.id.agent_prompt).text.toString().trim()
-        if (targetPackage.isEmpty()) {
-            showToast("Enter a target package")
-            return
-        }
         if (prompt.isEmpty()) {
             showToast("Enter a prompt")
             return
@@ -184,9 +180,13 @@ class MainActivity : Activity() {
         ensureCodexdRunningForAgent()
         thread {
             val result = runCatching {
+                val plan = AgentTaskPlanner.plan(
+                    context = this,
+                    userObjective = prompt,
+                    targetPackageOverride = targetPackageOverride.ifBlank { null },
+                )
                 agentSessionController.startDirectSession(
-                    targetPackage = targetPackage,
-                    prompt = prompt,
+                    plan = plan,
                     allowDetachedMode = true,
                 )
             }
@@ -195,8 +195,9 @@ class MainActivity : Activity() {
                 refreshAgentSessions()
             }
             result.onSuccess { sessionStart ->
-                focusedFrameworkSessionId = sessionStart.childSessionId
-                showToast("Started ${sessionStart.childSessionId} via ${sessionStart.geniePackage}")
+                focusedFrameworkSessionId = sessionStart.childSessionIds.firstOrNull()
+                val targetSummary = sessionStart.plannedTargets.joinToString(", ")
+                showToast("Started ${sessionStart.childSessionIds.size} Genie session(s) for $targetSummary via ${sessionStart.geniePackage}")
                 refreshAgentSessions()
             }
         }