Use Agent tools for Android planning

Co-authored-by: Codex <noreply@openai.com>

android/app/src/main/java/com/openai/codexd/AgentCodexAppServerClient.kt

@@ -42,14 +42,20 @@ object AgentCodexAppServerClient {
         context: Context,
         instructions: String,
         prompt: String,
+        dynamicTools: JSONArray? = null,
+        toolCallHandler: ((String, JSONObject) -> JSONObject)? = null,
     ): String = synchronized(lifecycleLock) {
         ensureStarted(context.applicationContext)
         activeRequests.incrementAndGet()
         try {
             notifications.clear()
-            val threadId = startThread(context.applicationContext, instructions)
+            val threadId = startThread(
+                context = context.applicationContext,
+                instructions = instructions,
+                dynamicTools = dynamicTools,
+            )
             startTurn(threadId, prompt)
-            waitForTurnCompletion()
+            waitForTurnCompletion(toolCallHandler)
         } finally {
             activeRequests.decrementAndGet()
         }
@@ -134,16 +140,21 @@ object AgentCodexAppServerClient {
     private fun startThread(
         context: Context,
         instructions: String,
+        dynamicTools: JSONArray?,
     ): String {
+        val params = JSONObject()
+            .put("approvalPolicy", "never")
+            .put("sandbox", "read-only")
+            .put("ephemeral", true)
+            .put("cwd", context.filesDir.absolutePath)
+            .put("serviceName", "android_agent")
+            .put("baseInstructions", instructions)
+        if (dynamicTools != null) {
+            params.put("dynamicTools", dynamicTools)
+        }
         val result = request(
             method = "thread/start",
-            params = JSONObject()
-                .put("approvalPolicy", "never")
-                .put("sandbox", "read-only")
-                .put("ephemeral", true)
-                .put("cwd", context.filesDir.absolutePath)
-                .put("serviceName", "android_agent")
-                .put("baseInstructions", instructions),
+            params = params,
         )
         return result.getJSONObject("thread").getString("id")
     }
@@ -167,7 +178,9 @@ object AgentCodexAppServerClient {
         )
     }
 
-    private fun waitForTurnCompletion(): String {
+    private fun waitForTurnCompletion(
+        toolCallHandler: ((String, JSONObject) -> JSONObject)?,
+    ): String {
         val streamedAgentMessages = mutableMapOf<String, StringBuilder>()
         var finalAgentMessage: String? = null
         val deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(REQUEST_TIMEOUT_MS)
@@ -182,7 +195,7 @@ object AgentCodexAppServerClient {
                 continue
             }
             if (notification.has("id") && notification.has("method")) {
-                rejectUnsupportedServerRequest(notification)
+                handleServerRequest(notification, toolCallHandler)
                 continue
             }
             val params = notification.optJSONObject("params") ?: JSONObject()
@@ -222,17 +235,67 @@ object AgentCodexAppServerClient {
         }
     }
 
-    private fun rejectUnsupportedServerRequest(message: JSONObject) {
+    private fun handleServerRequest(
+        message: JSONObject,
+        toolCallHandler: ((String, JSONObject) -> JSONObject)?,
+    ) {
         val requestId = message.opt("id") ?: return
         val method = message.optString("method", "unknown")
+        if (method != "item/tool/call") {
+            sendError(
+                requestId = requestId,
+                code = -32601,
+                message = "Unsupported Agent app-server request: $method",
+            )
+            return
+        }
+        if (toolCallHandler == null) {
+            sendError(
+                requestId = requestId,
+                code = -32601,
+                message = "No Agent tool handler registered for $method",
+            )
+            return
+        }
+        val params = message.optJSONObject("params") ?: JSONObject()
+        val toolName = params.optString("tool").trim()
+        val arguments = params.optJSONObject("arguments") ?: JSONObject()
+        val result = runCatching { toolCallHandler(toolName, arguments) }
+            .getOrElse { err ->
+                sendError(
+                    requestId = requestId,
+                    code = -32000,
+                    message = err.message ?: "Agent tool call failed",
+                )
+                return
+            }
+        sendResult(requestId, result)
+    }
+
+    private fun sendResult(
+        requestId: Any,
+        result: JSONObject,
+    ) {
+        sendMessage(
+            JSONObject()
+                .put("id", requestId)
+                .put("result", result),
+        )
+    }
+
+    private fun sendError(
+        requestId: Any,
+        code: Int,
+        message: String,
+    ) {
         sendMessage(
             JSONObject()
                 .put("id", requestId)
                 .put(
                     "error",
                     JSONObject()
-                        .put("code", -32601)
-                        .put("message", "Unsupported Agent app-server request: $method"),
+                        .put("code", code)
+                        .put("message", message),
                 ),
         )
     }

android/app/src/main/java/com/openai/codexd/AgentTaskPlanner.kt

@@ -22,15 +22,17 @@ data class AgentDelegationPlan(
 
 object AgentTaskPlanner {
     private const val MAX_LAUNCHABLE_APPS = 80
+    private const val LIST_LAUNCHABLE_APPS_TOOL = "android.apps.list_launchable"
     private val PLANNER_INSTRUCTIONS =
         """
         You are Codex acting as the Android Agent planner.
         The user interacts only with the Agent. Decide which installed Android packages should receive delegated Genie sessions.
+        Use the available Android app-list tool before selecting targets.
         Choose the fewest packages needed to complete the request.
         Return exactly one JSON object with this shape:
         {"targets":[{"packageName":"com.example.app","objective":"free-form delegated objective"}],"reason":"short explanation"}
         Rules:
-        - Use only package names from the provided installed-app list.
+        - Use only package names returned by the Android app-list tool.
         - `targets` must be non-empty.
         - Each delegated `objective` should be written for the child Genie, not the user.
         - Do not include markdown or code fences.
@@ -62,7 +64,11 @@ object AgentTaskPlanner {
         val planText = AgentCodexAppServerClient.requestText(
             context = context,
             instructions = PLANNER_INSTRUCTIONS,
-            prompt = buildPlannerPrompt(userObjective, launchableApps),
+            prompt = buildPlannerPrompt(userObjective),
+            dynamicTools = buildDynamicToolSpecs(),
+            toolCallHandler = { toolName, _ ->
+                handleToolCall(toolName, launchableApps)
+            },
         )
         return parsePlanResponse(
             responseText = planText,
@@ -106,22 +112,53 @@ object AgentTaskPlanner {
         )
     }
 
-    private fun buildPlannerPrompt(
-        userObjective: String,
-        launchableApps: List<InstalledLaunchableApp>,
-    ): String {
-        val appList = launchableApps.joinToString(separator = "\n") { app ->
-            "- ${app.label} (${app.packageName})"
-        }
+    private fun buildPlannerPrompt(userObjective: String): String {
         return """
             User objective:
             $userObjective
-
-            Installed launchable apps:
-            $appList
         """.trimIndent()
     }
 
+    private fun buildDynamicToolSpecs(): JSONArray {
+        return JSONArray().put(
+            JSONObject()
+                .put("name", LIST_LAUNCHABLE_APPS_TOOL)
+                .put(
+                    "description",
+                    "List the launchable Android packages currently installed on this device.",
+                )
+                .put(
+                    "inputSchema",
+                    JSONObject()
+                        .put("type", "object")
+                        .put("properties", JSONObject())
+                        .put("additionalProperties", false),
+                ),
+        )
+    }
+
+    private fun handleToolCall(
+        toolName: String,
+        launchableApps: List<InstalledLaunchableApp>,
+    ): JSONObject {
+        if (toolName != LIST_LAUNCHABLE_APPS_TOOL) {
+            throw IOException("Unsupported Agent planning tool: $toolName")
+        }
+        val appList = launchableApps.joinToString(separator = "\n") { app ->
+            "- ${app.label} (${app.packageName})"
+        }
+        return JSONObject()
+            .put("success", true)
+            .put(
+                "contentItems",
+                JSONArray().put(
+                    JSONObject()
+                        .put("type", "inputText")
+                        .put("text", "Launchable Android apps:\n$appList"),
+                ),
+            )
+    }
+
     private fun extractJsonObject(responseText: String): JSONObject {
         val start = responseText.indexOf('{')
         val end = responseText.lastIndexOf('}')