mirror of
https://github.com/openai/codex.git
synced 2026-04-24 06:35:50 +00:00
Track final Android target presentation state
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Iliyan Malchev
@@ -44,10 +44,16 @@ class AgentFrameworkToolBridge(
|
||||
continue
|
||||
}
|
||||
val objective = target.optString("objective").trim().ifEmpty { userObjective }
|
||||
val finalPresentationPolicy = target.optString("finalPresentationPolicy").trim()
|
||||
val defaultFinalPresentationPolicy = arguments.optString("finalPresentationPolicy").trim()
|
||||
add(
|
||||
AgentDelegationTarget(
|
||||
packageName = packageName,
|
||||
objective = objective,
|
||||
finalPresentationPolicy =
|
||||
SessionFinalPresentationPolicy.fromWireValue(finalPresentationPolicy)
|
||||
?: SessionFinalPresentationPolicy.fromWireValue(defaultFinalPresentationPolicy)
|
||||
?: SessionFinalPresentationPolicy.AGENT_CHOICE,
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -60,6 +66,13 @@ class AgentFrameworkToolBridge(
|
||||
}
|
||||
throw IOException("Framework session tool did not select an eligible target package")
|
||||
}
|
||||
val allowDetachedMode = arguments.optBoolean("allowDetachedMode", true)
|
||||
val detachedPolicyTargets = targets.filter { it.finalPresentationPolicy.requiresDetachedMode() }
|
||||
if (!allowDetachedMode && detachedPolicyTargets.isNotEmpty()) {
|
||||
throw IOException(
|
||||
"Framework session tool selected detached final presentation without allowDetachedMode: ${detachedPolicyTargets.joinToString(", ") { it.packageName }}",
|
||||
)
|
||||
}
|
||||
return StartDirectSessionRequest(
|
||||
plan = AgentDelegationPlan(
|
||||
originalObjective = userObjective,
|
||||
@@ -67,7 +80,7 @@ class AgentFrameworkToolBridge(
|
||||
rationale = arguments.optString("reason").trim().ifEmpty { null },
|
||||
usedOverride = false,
|
||||
),
|
||||
allowDetachedMode = arguments.optBoolean("allowDetachedMode", true),
|
||||
allowDetachedMode = allowDetachedMode,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -177,9 +190,20 @@ class AgentFrameworkToolBridge(
|
||||
"properties",
|
||||
JSONObject()
|
||||
.put("packageName", stringSchema("Installed target Android package name."))
|
||||
.put("objective", stringSchema("Delegated free-form objective for the child Genie.")),
|
||||
.put("objective", stringSchema("Delegated free-form objective for the child Genie."))
|
||||
.put(
|
||||
"finalPresentationPolicy",
|
||||
stringSchema(
|
||||
"Required final target presentation: ATTACHED, DETACHED_HIDDEN, DETACHED_SHOWN, or AGENT_CHOICE.",
|
||||
),
|
||||
),
|
||||
)
|
||||
.put(
|
||||
"required",
|
||||
JSONArray()
|
||||
.put("packageName")
|
||||
.put("finalPresentationPolicy"),
|
||||
)
|
||||
.put("required", JSONArray().put("packageName"))
|
||||
.put("additionalProperties", false),
|
||||
),
|
||||
)
|
||||
@@ -272,7 +296,12 @@ class AgentFrameworkToolBridge(
|
||||
.put("parentSessionId", session.parentSessionId)
|
||||
.put("targetPackage", session.targetPackage)
|
||||
.put("state", session.stateLabel)
|
||||
.put("targetDetached", session.targetDetached),
|
||||
.put("targetDetached", session.targetDetached)
|
||||
.put("targetPresentation", session.targetPresentationLabel)
|
||||
.put(
|
||||
"requiredFinalPresentation",
|
||||
session.requiredFinalPresentationPolicy?.wireValue,
|
||||
),
|
||||
)
|
||||
}
|
||||
return JSONObject()
|
||||
|
||||
@@ -16,6 +16,8 @@ data class ParentSessionChildSummary(
|
||||
val sessionId: String,
|
||||
val targetPackage: String?,
|
||||
val state: Int,
|
||||
val targetPresentation: Int,
|
||||
val requiredFinalPresentationPolicy: SessionFinalPresentationPolicy?,
|
||||
val latestResult: String?,
|
||||
val latestError: String?,
|
||||
)
|
||||
@@ -24,26 +26,67 @@ data class ParentSessionRollup(
|
||||
val state: Int,
|
||||
val resultMessage: String?,
|
||||
val errorMessage: String?,
|
||||
val sessionsToAttach: List<String>,
|
||||
)
|
||||
|
||||
object AgentParentSessionAggregator {
|
||||
fun rollup(childSessions: List<ParentSessionChildSummary>): ParentSessionRollup {
|
||||
val state = computeParentState(childSessions.map(ParentSessionChildSummary::state))
|
||||
return when (state) {
|
||||
val baseState = computeParentState(childSessions.map(ParentSessionChildSummary::state))
|
||||
if (
|
||||
baseState == AgentSessionInfo.STATE_CREATED ||
|
||||
baseState == AgentSessionInfo.STATE_RUNNING ||
|
||||
baseState == AgentSessionInfo.STATE_WAITING_FOR_USER ||
|
||||
baseState == AgentSessionInfo.STATE_QUEUED
|
||||
) {
|
||||
return ParentSessionRollup(
|
||||
state = baseState,
|
||||
resultMessage = null,
|
||||
errorMessage = null,
|
||||
sessionsToAttach = emptyList(),
|
||||
)
|
||||
}
|
||||
val terminalPresentationMismatches = childSessions.mapNotNull { childSession ->
|
||||
childSession.presentationMismatch()
|
||||
}
|
||||
val sessionsToAttach = terminalPresentationMismatches
|
||||
.filter { it.requiredPolicy == SessionFinalPresentationPolicy.ATTACHED }
|
||||
.map(PresentationMismatch::sessionId)
|
||||
val blockingMismatches = terminalPresentationMismatches
|
||||
.filterNot { it.requiredPolicy == SessionFinalPresentationPolicy.ATTACHED }
|
||||
if (sessionsToAttach.isNotEmpty() && baseState == AgentSessionInfo.STATE_COMPLETED) {
|
||||
return ParentSessionRollup(
|
||||
state = AgentSessionInfo.STATE_RUNNING,
|
||||
resultMessage = null,
|
||||
errorMessage = null,
|
||||
sessionsToAttach = sessionsToAttach,
|
||||
)
|
||||
}
|
||||
if (blockingMismatches.isNotEmpty()) {
|
||||
return ParentSessionRollup(
|
||||
state = AgentSessionInfo.STATE_FAILED,
|
||||
resultMessage = null,
|
||||
errorMessage = buildPresentationMismatchError(blockingMismatches),
|
||||
sessionsToAttach = emptyList(),
|
||||
)
|
||||
}
|
||||
return when (baseState) {
|
||||
AgentSessionInfo.STATE_COMPLETED -> ParentSessionRollup(
|
||||
state = state,
|
||||
state = baseState,
|
||||
resultMessage = buildParentResult(childSessions),
|
||||
errorMessage = null,
|
||||
sessionsToAttach = emptyList(),
|
||||
)
|
||||
AgentSessionInfo.STATE_FAILED -> ParentSessionRollup(
|
||||
state = state,
|
||||
state = baseState,
|
||||
resultMessage = null,
|
||||
errorMessage = buildParentError(childSessions),
|
||||
sessionsToAttach = emptyList(),
|
||||
)
|
||||
else -> ParentSessionRollup(
|
||||
state = state,
|
||||
state = baseState,
|
||||
resultMessage = null,
|
||||
errorMessage = null,
|
||||
sessionsToAttach = emptyList(),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -106,6 +149,20 @@ object AgentParentSessionAggregator {
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildPresentationMismatchError(mismatches: List<PresentationMismatch>): String {
|
||||
return buildString {
|
||||
append("Delegated session completed without the required final presentation")
|
||||
mismatches.forEach { mismatch ->
|
||||
append("; ")
|
||||
append(mismatch.targetPackage ?: mismatch.sessionId)
|
||||
append(": required ")
|
||||
append(mismatch.requiredPolicy.wireValue)
|
||||
append(", actual ")
|
||||
append(targetPresentationToString(mismatch.actualPresentation))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun stateToString(state: Int): String {
|
||||
return when (state) {
|
||||
AgentSessionInfo.STATE_CREATED -> "CREATED"
|
||||
@@ -118,4 +175,24 @@ object AgentParentSessionAggregator {
|
||||
else -> state.toString()
|
||||
}
|
||||
}
|
||||
|
||||
private fun ParentSessionChildSummary.presentationMismatch(): PresentationMismatch? {
|
||||
val requiredPolicy = requiredFinalPresentationPolicy ?: return null
|
||||
if (state != AgentSessionInfo.STATE_COMPLETED || requiredPolicy.matches(targetPresentation)) {
|
||||
return null
|
||||
}
|
||||
return PresentationMismatch(
|
||||
sessionId = sessionId,
|
||||
targetPackage = targetPackage,
|
||||
requiredPolicy = requiredPolicy,
|
||||
actualPresentation = targetPresentation,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private data class PresentationMismatch(
|
||||
val sessionId: String,
|
||||
val targetPackage: String?,
|
||||
val requiredPolicy: SessionFinalPresentationPolicy,
|
||||
val actualPresentation: Int,
|
||||
)
|
||||
|
||||
@@ -21,6 +21,7 @@ class AgentSessionController(context: Context) {
|
||||
}
|
||||
|
||||
private val agentManager = context.getSystemService(AgentManager::class.java)
|
||||
private val presentationPolicyStore = SessionPresentationPolicyStore(context)
|
||||
|
||||
fun isAvailable(): Boolean = agentManager != null
|
||||
|
||||
@@ -49,7 +50,9 @@ class AgentSessionController(context: Context) {
|
||||
val manager = agentManager ?: return AgentSnapshot.unavailable
|
||||
val roleHolders = manager.getGenieRoleHolders(currentUserId())
|
||||
val selectedGeniePackage = selectGeniePackage(roleHolders)
|
||||
var sessionDetails = manager.getSessions(currentUserId()).map { session ->
|
||||
val sessions = manager.getSessions(currentUserId())
|
||||
presentationPolicyStore.prunePolicies(sessions.map { it.sessionId }.toSet())
|
||||
var sessionDetails = sessions.map { session ->
|
||||
AgentSessionDetails(
|
||||
sessionId = session.sessionId,
|
||||
parentSessionId = session.parentSessionId,
|
||||
@@ -57,7 +60,10 @@ class AgentSessionController(context: Context) {
|
||||
anchor = session.anchor,
|
||||
state = session.state,
|
||||
stateLabel = stateToString(session.state),
|
||||
targetPresentation = session.targetPresentation,
|
||||
targetPresentationLabel = targetPresentationToString(session.targetPresentation),
|
||||
targetDetached = session.isTargetDetached,
|
||||
requiredFinalPresentationPolicy = presentationPolicyStore.getPolicy(session.sessionId),
|
||||
latestQuestion = null,
|
||||
latestResult = null,
|
||||
latestError = null,
|
||||
@@ -105,6 +111,10 @@ class AgentSessionController(context: Context) {
|
||||
allowDetachedMode: Boolean,
|
||||
): SessionStartResult {
|
||||
val manager = requireAgentManager()
|
||||
val detachedPolicyTargets = plan.targets.filter { it.finalPresentationPolicy.requiresDetachedMode() }
|
||||
check(allowDetachedMode || detachedPolicyTargets.isEmpty()) {
|
||||
"Detached final presentation requires detached mode for ${detachedPolicyTargets.joinToString(", ") { it.packageName }}"
|
||||
}
|
||||
val geniePackage = selectGeniePackage(manager.getGenieRoleHolders(currentUserId()))
|
||||
?: throw IllegalStateException("No GENIE role holder configured")
|
||||
val parentSession = manager.createDirectSession(currentUserId())
|
||||
@@ -120,14 +130,15 @@ class AgentSessionController(context: Context) {
|
||||
plan.targets.forEach { target ->
|
||||
val childSession = manager.createChildSession(parentSession.sessionId, target.packageName)
|
||||
childSessionIds += childSession.sessionId
|
||||
presentationPolicyStore.savePolicy(childSession.sessionId, target.finalPresentationPolicy)
|
||||
manager.publishTrace(
|
||||
parentSession.sessionId,
|
||||
"Created child session ${childSession.sessionId} for ${target.packageName}.",
|
||||
"Created child session ${childSession.sessionId} for ${target.packageName} with required final presentation ${target.finalPresentationPolicy.wireValue}.",
|
||||
)
|
||||
manager.startGenieSession(
|
||||
childSession.sessionId,
|
||||
geniePackage,
|
||||
target.objective,
|
||||
buildDelegatedPrompt(target),
|
||||
allowDetachedMode,
|
||||
)
|
||||
}
|
||||
@@ -140,6 +151,7 @@ class AgentSessionController(context: Context) {
|
||||
} catch (err: RuntimeException) {
|
||||
childSessionIds.forEach { childSessionId ->
|
||||
runCatching { manager.cancelSession(childSessionId) }
|
||||
presentationPolicyStore.removePolicy(childSessionId)
|
||||
}
|
||||
runCatching { manager.cancelSession(parentSession.sessionId) }
|
||||
throw err
|
||||
@@ -269,6 +281,15 @@ class AgentSessionController(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildDelegatedPrompt(target: AgentDelegationTarget): String {
|
||||
return buildString {
|
||||
appendLine(target.objective)
|
||||
appendLine()
|
||||
appendLine("Required final target presentation: ${target.finalPresentationPolicy.wireValue}")
|
||||
append(target.finalPresentationPolicy.promptGuidance())
|
||||
}.trim()
|
||||
}
|
||||
|
||||
private fun findLastEventMessage(events: List<AgentSessionEvent>, type: Int): String? {
|
||||
for (index in events.indices.reversed()) {
|
||||
val event = events[index]
|
||||
@@ -353,6 +374,7 @@ class AgentSessionController(context: Context) {
|
||||
AgentSessionEvent.TYPE_RESULT -> "Result"
|
||||
AgentSessionEvent.TYPE_ERROR -> "Error"
|
||||
AgentSessionEvent.TYPE_POLICY -> "Policy"
|
||||
AgentSessionEvent.TYPE_DETACHED_ACTION -> "DetachedAction"
|
||||
AgentSessionEvent.TYPE_ANSWER -> "Answer"
|
||||
else -> "Event($type)"
|
||||
}
|
||||
@@ -415,7 +437,10 @@ data class AgentSessionDetails(
|
||||
val anchor: Int,
|
||||
val state: Int,
|
||||
val stateLabel: String,
|
||||
val targetPresentation: Int,
|
||||
val targetPresentationLabel: String,
|
||||
val targetDetached: Boolean,
|
||||
val requiredFinalPresentationPolicy: SessionFinalPresentationPolicy?,
|
||||
val latestQuestion: String?,
|
||||
val latestResult: String?,
|
||||
val latestError: String?,
|
||||
|
||||
@@ -10,6 +10,7 @@ import org.json.JSONTokener
|
||||
data class AgentDelegationTarget(
|
||||
val packageName: String,
|
||||
val objective: String,
|
||||
val finalPresentationPolicy: SessionFinalPresentationPolicy,
|
||||
)
|
||||
|
||||
data class AgentDelegationPlan(
|
||||
@@ -37,7 +38,8 @@ object AgentTaskPlanner {
|
||||
"targets": [
|
||||
{
|
||||
"packageName": "installed.package",
|
||||
"objective": "free-form delegated objective for the child Genie"
|
||||
"objective": "free-form delegated objective for the child Genie",
|
||||
"finalPresentationPolicy": "ATTACHED | DETACHED_HIDDEN | DETACHED_SHOWN | AGENT_CHOICE"
|
||||
}
|
||||
],
|
||||
"reason": "short rationale",
|
||||
@@ -47,6 +49,11 @@ object AgentTaskPlanner {
|
||||
- Choose the fewest packages needed to complete the request.
|
||||
- `targets` must be non-empty.
|
||||
- Each delegated `objective` should be written for the child Genie, not the user.
|
||||
- Each target must include `finalPresentationPolicy`.
|
||||
- Use `ATTACHED` when the user wants the target left on the main screen or explicitly visible to them.
|
||||
- Use `DETACHED_SHOWN` when the target should remain visible but stay detached.
|
||||
- Use `DETACHED_HIDDEN` when the target should complete in the background without remaining visible.
|
||||
- Use `AGENT_CHOICE` only when the final presentation state does not matter.
|
||||
- Stop after at most 6 shell commands.
|
||||
- Prefer direct package-manager commands over grepping large package lists.
|
||||
- Verify each chosen package by inspecting its package dump or query-activities output before returning it.
|
||||
@@ -76,9 +83,28 @@ object AgentTaskPlanner {
|
||||
"properties",
|
||||
JSONObject()
|
||||
.put("packageName", JSONObject().put("type", "string"))
|
||||
.put("objective", JSONObject().put("type", "string")),
|
||||
.put("objective", JSONObject().put("type", "string"))
|
||||
.put(
|
||||
"finalPresentationPolicy",
|
||||
JSONObject()
|
||||
.put("type", "string")
|
||||
.put(
|
||||
"enum",
|
||||
JSONArray()
|
||||
.put(SessionFinalPresentationPolicy.ATTACHED.wireValue)
|
||||
.put(SessionFinalPresentationPolicy.DETACHED_HIDDEN.wireValue)
|
||||
.put(SessionFinalPresentationPolicy.DETACHED_SHOWN.wireValue)
|
||||
.put(SessionFinalPresentationPolicy.AGENT_CHOICE.wireValue),
|
||||
),
|
||||
),
|
||||
)
|
||||
.put(
|
||||
"required",
|
||||
JSONArray()
|
||||
.put("packageName")
|
||||
.put("objective")
|
||||
.put("finalPresentationPolicy"),
|
||||
)
|
||||
.put("required", JSONArray().put("packageName").put("objective"))
|
||||
.put("additionalProperties", false),
|
||||
),
|
||||
)
|
||||
@@ -93,6 +119,7 @@ object AgentTaskPlanner {
|
||||
userObjective: String,
|
||||
targetPackageOverride: String?,
|
||||
allowDetachedMode: Boolean,
|
||||
finalPresentationPolicyOverride: SessionFinalPresentationPolicy? = null,
|
||||
sessionController: AgentSessionController,
|
||||
requestUserInputHandler: ((JSONArray) -> JSONObject)? = null,
|
||||
): SessionStartResult {
|
||||
@@ -105,6 +132,8 @@ object AgentTaskPlanner {
|
||||
AgentDelegationTarget(
|
||||
packageName = targetPackageOverride,
|
||||
objective = userObjective,
|
||||
finalPresentationPolicy =
|
||||
finalPresentationPolicyOverride ?: SessionFinalPresentationPolicy.AGENT_CHOICE,
|
||||
),
|
||||
),
|
||||
rationale = "Using explicit target package override.",
|
||||
|
||||
@@ -38,6 +38,7 @@ class CodexAgentService : AgentService() {
|
||||
|
||||
private val agentManager by lazy { getSystemService(AgentManager::class.java) }
|
||||
private val sessionController by lazy { AgentSessionController(this) }
|
||||
private val presentationPolicyStore by lazy { SessionPresentationPolicyStore(this) }
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
@@ -73,6 +74,7 @@ class CodexAgentService : AgentService() {
|
||||
Log.i(TAG, "onSessionRemoved sessionId=$sessionId")
|
||||
AgentSessionBridgeServer.closeSession(sessionId)
|
||||
AgentQuestionNotifier.cancel(this, sessionId)
|
||||
presentationPolicyStore.removePolicy(sessionId)
|
||||
handledGenieQuestions.removeIf { it.startsWith("$sessionId:") }
|
||||
handledBridgeRequests.removeIf { it.startsWith("$sessionId:") }
|
||||
pendingGenieQuestions.removeIf { it.startsWith("$sessionId:") }
|
||||
@@ -90,9 +92,9 @@ class CodexAgentService : AgentService() {
|
||||
thread(name = "CodexAgentParentRollup-$parentSessionId") {
|
||||
try {
|
||||
runCatching {
|
||||
rollUpParentSession(parentSessionId)
|
||||
}.onFailure { err ->
|
||||
Log.w(TAG, "Parent session roll-up failed for $parentSessionId", err)
|
||||
rollUpParentSession(parentSessionId)
|
||||
}.onFailure { err ->
|
||||
Log.w(TAG, "Parent session roll-up failed for $parentSessionId", err)
|
||||
}
|
||||
} finally {
|
||||
pendingParentRollups.remove(parentSessionId)
|
||||
@@ -118,11 +120,24 @@ class CodexAgentService : AgentService() {
|
||||
sessionId = childSession.sessionId,
|
||||
targetPackage = childSession.targetPackage,
|
||||
state = childSession.state,
|
||||
targetPresentation = childSession.targetPresentation,
|
||||
requiredFinalPresentationPolicy = presentationPolicyStore.getPolicy(childSession.sessionId),
|
||||
latestResult = findLastEventMessage(events, AgentSessionEvent.TYPE_RESULT),
|
||||
latestError = findLastEventMessage(events, AgentSessionEvent.TYPE_ERROR),
|
||||
)
|
||||
},
|
||||
)
|
||||
rollup.sessionsToAttach.forEach { childSessionId ->
|
||||
runCatching {
|
||||
manager.attachTarget(childSessionId)
|
||||
manager.publishTrace(
|
||||
parentSessionId,
|
||||
"Requested attach for $childSessionId to satisfy the required final presentation policy.",
|
||||
)
|
||||
}.onFailure { err ->
|
||||
Log.w(TAG, "Failed to attach target for $childSessionId", err)
|
||||
}
|
||||
}
|
||||
if (parentSession.state != rollup.state) {
|
||||
runCatching {
|
||||
manager.updateSessionState(parentSessionId, rollup.state)
|
||||
@@ -426,6 +441,7 @@ class CodexAgentService : AgentService() {
|
||||
AgentSessionEvent.TYPE_RESULT -> "Result"
|
||||
AgentSessionEvent.TYPE_ERROR -> "Error"
|
||||
AgentSessionEvent.TYPE_POLICY -> "Policy"
|
||||
AgentSessionEvent.TYPE_DETACHED_ACTION -> "DetachedAction"
|
||||
AgentSessionEvent.TYPE_ANSWER -> "Answer"
|
||||
else -> "Event($type)"
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ class MainActivity : Activity() {
|
||||
"com.openai.codex.agent.action.DEBUG_CANCEL_ALL_AGENT_SESSIONS"
|
||||
private const val EXTRA_DEBUG_PROMPT = "prompt"
|
||||
private const val EXTRA_DEBUG_TARGET_PACKAGE = "targetPackage"
|
||||
private const val EXTRA_DEBUG_FINAL_PRESENTATION_POLICY = "finalPresentationPolicy"
|
||||
}
|
||||
|
||||
@Volatile
|
||||
@@ -153,10 +154,19 @@ class MainActivity : Activity() {
|
||||
return
|
||||
}
|
||||
val targetPackageOverride = intent.getStringExtra(EXTRA_DEBUG_TARGET_PACKAGE)?.trim()
|
||||
val finalPresentationPolicyOverride = SessionFinalPresentationPolicy.fromWireValue(
|
||||
intent.getStringExtra(EXTRA_DEBUG_FINAL_PRESENTATION_POLICY),
|
||||
)
|
||||
findViewById<EditText>(R.id.agent_prompt).setText(prompt)
|
||||
findViewById<EditText>(R.id.agent_target_package).setText(targetPackageOverride.orEmpty())
|
||||
Log.i(TAG, "Handling debug start intent override=$targetPackageOverride prompt=${prompt.take(160)}")
|
||||
startDirectAgentSession(findViewById(R.id.agent_start_button))
|
||||
Log.i(
|
||||
TAG,
|
||||
"Handling debug start intent override=$targetPackageOverride finalPresentationPolicy=${finalPresentationPolicyOverride?.wireValue} prompt=${prompt.take(160)}",
|
||||
)
|
||||
startDirectAgentSession(
|
||||
view = findViewById(R.id.agent_start_button),
|
||||
finalPresentationPolicyOverride = finalPresentationPolicyOverride,
|
||||
)
|
||||
intent.action = null
|
||||
}
|
||||
|
||||
@@ -211,13 +221,23 @@ class MainActivity : Activity() {
|
||||
}
|
||||
|
||||
fun startDirectAgentSession(@Suppress("UNUSED_PARAMETER") view: View) {
|
||||
startDirectAgentSession(view, null)
|
||||
}
|
||||
|
||||
private fun startDirectAgentSession(
|
||||
@Suppress("UNUSED_PARAMETER") view: View,
|
||||
finalPresentationPolicyOverride: SessionFinalPresentationPolicy?,
|
||||
) {
|
||||
val targetPackageOverride = findViewById<EditText>(R.id.agent_target_package).text.toString().trim()
|
||||
val prompt = findViewById<EditText>(R.id.agent_prompt).text.toString().trim()
|
||||
if (prompt.isEmpty()) {
|
||||
showToast("Enter a prompt")
|
||||
return
|
||||
}
|
||||
Log.i(TAG, "startDirectAgentSession override=$targetPackageOverride prompt=${prompt.take(160)}")
|
||||
Log.i(
|
||||
TAG,
|
||||
"startDirectAgentSession override=$targetPackageOverride finalPresentationPolicy=${finalPresentationPolicyOverride?.wireValue} prompt=${prompt.take(160)}",
|
||||
)
|
||||
thread {
|
||||
val result = runCatching {
|
||||
AgentTaskPlanner.startSession(
|
||||
@@ -225,6 +245,7 @@ class MainActivity : Activity() {
|
||||
userObjective = prompt,
|
||||
targetPackageOverride = targetPackageOverride.ifBlank { null },
|
||||
allowDetachedMode = true,
|
||||
finalPresentationPolicyOverride = finalPresentationPolicyOverride,
|
||||
sessionController = agentSessionController,
|
||||
requestUserInputHandler = { questions ->
|
||||
AgentUserInputPrompter.promptForAnswers(this, questions)
|
||||
@@ -493,7 +514,8 @@ class MainActivity : Activity() {
|
||||
answerButton.visibility = if (isWaitingForUser) View.VISIBLE else View.GONE
|
||||
questionView.text = waitingQuestion ?: ""
|
||||
|
||||
val showAttachButton = selectedSession?.targetDetached == true
|
||||
val showAttachButton =
|
||||
selectedSession?.targetPresentation != AgentSessionInfo.TARGET_PRESENTATION_ATTACHED
|
||||
attachButton.visibility = if (showAttachButton) View.VISIBLE else View.GONE
|
||||
attachButton.isEnabled = showAttachButton
|
||||
|
||||
@@ -517,7 +539,10 @@ class MainActivity : Activity() {
|
||||
append("Session: ${selectedSession.sessionId}\n")
|
||||
append("State: ${selectedSession.stateLabel}\n")
|
||||
append("Target: ${selectedSession.targetPackage ?: "direct-agent"}\n")
|
||||
append("Detached target: ${selectedSession.targetDetached}\n")
|
||||
append("Target presentation: ${selectedSession.targetPresentationLabel}\n")
|
||||
selectedSession.requiredFinalPresentationPolicy?.let { policy ->
|
||||
append("Required final presentation: ${policy.wireValue}\n")
|
||||
}
|
||||
val parentSessionId = snapshot.parentSession?.sessionId
|
||||
if (parentSessionId != null) {
|
||||
append("Parent: $parentSessionId\n")
|
||||
@@ -545,8 +570,9 @@ class MainActivity : Activity() {
|
||||
val detail = session.latestQuestion ?: session.latestResult ?: session.latestError ?: session.latestTrace
|
||||
buildString {
|
||||
append("$marker $role ${session.stateLabel} ${session.targetPackage ?: "direct-agent"}")
|
||||
if (session.targetDetached) {
|
||||
append(" [detached]")
|
||||
append(" [${session.targetPresentationLabel}]")
|
||||
session.requiredFinalPresentationPolicy?.let { policy ->
|
||||
append(" -> ${policy.wireValue}")
|
||||
}
|
||||
append("\n ${session.sessionId}")
|
||||
if (!detail.isNullOrEmpty()) {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package com.openai.codex.agent
|
||||
|
||||
import android.app.agent.AgentSessionInfo
|
||||
import java.io.IOException
|
||||
|
||||
enum class SessionFinalPresentationPolicy(
|
||||
val wireValue: String,
|
||||
val description: String,
|
||||
) {
|
||||
ATTACHED(
|
||||
wireValue = "ATTACHED",
|
||||
description = "Finish with the target attached to the main user-facing display/task stack.",
|
||||
),
|
||||
DETACHED_HIDDEN(
|
||||
wireValue = "DETACHED_HIDDEN",
|
||||
description = "Finish with the target still detached and hidden from view.",
|
||||
),
|
||||
DETACHED_SHOWN(
|
||||
wireValue = "DETACHED_SHOWN",
|
||||
description = "Finish with the target detached but visibly shown through the detached host.",
|
||||
),
|
||||
AGENT_CHOICE(
|
||||
wireValue = "AGENT_CHOICE",
|
||||
description = "The Agent does not require a specific final presentation state for this target.",
|
||||
),
|
||||
;
|
||||
|
||||
fun matches(actualPresentation: Int): Boolean {
|
||||
return when (this) {
|
||||
ATTACHED -> actualPresentation == AgentSessionInfo.TARGET_PRESENTATION_ATTACHED
|
||||
DETACHED_HIDDEN -> {
|
||||
actualPresentation == AgentSessionInfo.TARGET_PRESENTATION_DETACHED_HIDDEN
|
||||
}
|
||||
DETACHED_SHOWN -> {
|
||||
actualPresentation == AgentSessionInfo.TARGET_PRESENTATION_DETACHED_SHOWN
|
||||
}
|
||||
AGENT_CHOICE -> true
|
||||
}
|
||||
}
|
||||
|
||||
fun requiresDetachedMode(): Boolean {
|
||||
return when (this) {
|
||||
DETACHED_HIDDEN, DETACHED_SHOWN -> true
|
||||
ATTACHED, AGENT_CHOICE -> false
|
||||
}
|
||||
}
|
||||
|
||||
fun promptGuidance(): String {
|
||||
return when (this) {
|
||||
ATTACHED -> {
|
||||
"Before reporting success, ensure the target is ATTACHED to the primary user-facing display. Detached-only visibility is not sufficient."
|
||||
}
|
||||
DETACHED_HIDDEN -> {
|
||||
"Before reporting success, ensure the target remains DETACHED_HIDDEN. Do not attach it or leave it shown."
|
||||
}
|
||||
DETACHED_SHOWN -> {
|
||||
"Before reporting success, ensure the target remains DETACHED_SHOWN. It should stay detached but visibly shown through the detached host."
|
||||
}
|
||||
AGENT_CHOICE -> {
|
||||
"Choose the final target presentation state yourself and describe the final state accurately in your result."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun fromWireValue(value: String?): SessionFinalPresentationPolicy? {
|
||||
val normalized = value?.trim().orEmpty()
|
||||
if (normalized.isEmpty()) {
|
||||
return null
|
||||
}
|
||||
return entries.firstOrNull { it.wireValue.equals(normalized, ignoreCase = true) }
|
||||
}
|
||||
|
||||
fun requireFromWireValue(
|
||||
value: String?,
|
||||
fieldName: String,
|
||||
): SessionFinalPresentationPolicy {
|
||||
return fromWireValue(value)
|
||||
?: throw IOException("Unsupported $fieldName: ${value?.trim().orEmpty()}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AgentTargetPresentationValues {
|
||||
const val ATTACHED = AgentSessionInfo.TARGET_PRESENTATION_ATTACHED
|
||||
const val DETACHED_HIDDEN = AgentSessionInfo.TARGET_PRESENTATION_DETACHED_HIDDEN
|
||||
const val DETACHED_SHOWN = AgentSessionInfo.TARGET_PRESENTATION_DETACHED_SHOWN
|
||||
}
|
||||
|
||||
fun targetPresentationToString(targetPresentation: Int): String {
|
||||
return when (targetPresentation) {
|
||||
AgentTargetPresentationValues.ATTACHED -> "ATTACHED"
|
||||
AgentTargetPresentationValues.DETACHED_HIDDEN -> "DETACHED_HIDDEN"
|
||||
AgentTargetPresentationValues.DETACHED_SHOWN -> "DETACHED_SHOWN"
|
||||
else -> targetPresentation.toString()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.openai.codex.agent
|
||||
|
||||
import android.content.Context
|
||||
|
||||
class SessionPresentationPolicyStore(
|
||||
context: Context,
|
||||
) {
|
||||
companion object {
|
||||
private const val PREFS_NAME = "codex_session_presentation_policies"
|
||||
}
|
||||
|
||||
private val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
|
||||
|
||||
fun savePolicy(
|
||||
sessionId: String,
|
||||
policy: SessionFinalPresentationPolicy,
|
||||
) {
|
||||
prefs.edit().putString(sessionId, policy.wireValue).apply()
|
||||
}
|
||||
|
||||
fun getPolicy(sessionId: String): SessionFinalPresentationPolicy? {
|
||||
return SessionFinalPresentationPolicy.fromWireValue(
|
||||
prefs.getString(sessionId, null),
|
||||
)
|
||||
}
|
||||
|
||||
fun removePolicy(sessionId: String) {
|
||||
prefs.edit().remove(sessionId).apply()
|
||||
}
|
||||
|
||||
fun prunePolicies(activeSessionIds: Set<String>) {
|
||||
val staleSessionIds = prefs.all.keys - activeSessionIds
|
||||
if (staleSessionIds.isEmpty()) {
|
||||
return
|
||||
}
|
||||
prefs.edit().apply {
|
||||
staleSessionIds.forEach(::remove)
|
||||
}.apply()
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,8 @@ class AgentFrameworkToolBridgeTest {
|
||||
"targets": [
|
||||
{
|
||||
"packageName": "com.android.deskclock",
|
||||
"objective": "Start the requested timer in Clock."
|
||||
"objective": "Start the requested timer in Clock.",
|
||||
"finalPresentationPolicy": "ATTACHED"
|
||||
}
|
||||
],
|
||||
"reason": "Clock is the installed timer app.",
|
||||
@@ -34,6 +35,10 @@ class AgentFrameworkToolBridgeTest {
|
||||
assertEquals(1, request.plan.targets.size)
|
||||
assertEquals("com.android.deskclock", request.plan.targets.single().packageName)
|
||||
assertEquals("Start the requested timer in Clock.", request.plan.targets.single().objective)
|
||||
assertEquals(
|
||||
SessionFinalPresentationPolicy.ATTACHED,
|
||||
request.plan.targets.single().finalPresentationPolicy,
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -55,6 +60,10 @@ class AgentFrameworkToolBridgeTest {
|
||||
)
|
||||
|
||||
assertEquals("Start a 5-minute timer.", request.plan.targets.single().objective)
|
||||
assertEquals(
|
||||
SessionFinalPresentationPolicy.AGENT_CHOICE,
|
||||
request.plan.targets.single().finalPresentationPolicy,
|
||||
)
|
||||
assertEquals(true, request.allowDetachedMode)
|
||||
}
|
||||
|
||||
@@ -68,7 +77,8 @@ class AgentFrameworkToolBridgeTest {
|
||||
"targets": [
|
||||
{
|
||||
"packageName": "com.unknown.app",
|
||||
"objective": "Do the task."
|
||||
"objective": "Do the task.",
|
||||
"finalPresentationPolicy": "AGENT_CHOICE"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -86,4 +96,32 @@ class AgentFrameworkToolBridgeTest {
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun parseStartDirectSessionArgumentsRejectsDetachedPresentationWithoutDetachedMode() {
|
||||
val err = runCatching {
|
||||
AgentFrameworkToolBridge.parseStartDirectSessionArguments(
|
||||
arguments = JSONObject(
|
||||
"""
|
||||
{
|
||||
"targets": [
|
||||
{
|
||||
"packageName": "com.android.deskclock",
|
||||
"finalPresentationPolicy": "DETACHED_SHOWN"
|
||||
}
|
||||
],
|
||||
"allowDetachedMode": false
|
||||
}
|
||||
""".trimIndent(),
|
||||
),
|
||||
userObjective = "Keep Clock visible in detached mode.",
|
||||
isEligibleTargetPackage = linkedSetOf("com.android.deskclock")::contains,
|
||||
)
|
||||
}.exceptionOrNull()
|
||||
|
||||
assertTrue(err is java.io.IOException)
|
||||
assertEquals(
|
||||
"Framework session tool selected detached final presentation without allowDetachedMode: com.android.deskclock",
|
||||
err?.message,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,84 +1,76 @@
|
||||
package com.openai.codex.agent
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Test
|
||||
|
||||
class AgentParentSessionAggregatorTest {
|
||||
@Test
|
||||
fun rollupReturnsCompletedSummaryWhenChildrenComplete() {
|
||||
fun rollupRequestsAttachWhenAttachedPresentationIsRequired() {
|
||||
val rollup = AgentParentSessionAggregator.rollup(
|
||||
listOf(
|
||||
ParentSessionChildSummary(
|
||||
sessionId = "child-1",
|
||||
targetPackage = "com.android.deskclock",
|
||||
state = AgentSessionStateValues.COMPLETED,
|
||||
latestResult = "Alarm set for 2:07 PM.",
|
||||
targetPresentation = AgentTargetPresentationValues.DETACHED_SHOWN,
|
||||
requiredFinalPresentationPolicy = SessionFinalPresentationPolicy.ATTACHED,
|
||||
latestResult = "Started the stopwatch.",
|
||||
latestError = null,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
assertEquals(AgentSessionStateValues.COMPLETED, rollup.state)
|
||||
assertEquals(
|
||||
"Completed delegated session; com.android.deskclock: Alarm set for 2:07 PM.",
|
||||
rollup.resultMessage,
|
||||
)
|
||||
assertNull(rollup.errorMessage)
|
||||
assertEquals(AgentSessionStateValues.RUNNING, rollup.state)
|
||||
assertEquals(listOf("child-1"), rollup.sessionsToAttach)
|
||||
assertEquals(null, rollup.resultMessage)
|
||||
assertEquals(null, rollup.errorMessage)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rollupReturnsWaitingWhenAnyChildWaitsForUser() {
|
||||
fun rollupFailsWhenDetachedShownIsRequiredButTargetIsHidden() {
|
||||
val rollup = AgentParentSessionAggregator.rollup(
|
||||
listOf(
|
||||
ParentSessionChildSummary(
|
||||
sessionId = "child-1",
|
||||
targetPackage = "com.android.deskclock",
|
||||
state = AgentSessionStateValues.WAITING_FOR_USER,
|
||||
latestResult = null,
|
||||
latestError = null,
|
||||
),
|
||||
ParentSessionChildSummary(
|
||||
sessionId = "child-2",
|
||||
targetPackage = "com.android.settings",
|
||||
state = AgentSessionStateValues.COMPLETED,
|
||||
latestResult = "Completed task.",
|
||||
latestError = null,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
assertEquals(AgentSessionStateValues.WAITING_FOR_USER, rollup.state)
|
||||
assertNull(rollup.resultMessage)
|
||||
assertNull(rollup.errorMessage)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rollupReturnsFailedSummaryWhenAnyChildFails() {
|
||||
val rollup = AgentParentSessionAggregator.rollup(
|
||||
listOf(
|
||||
ParentSessionChildSummary(
|
||||
sessionId = "child-1",
|
||||
targetPackage = "com.android.deskclock",
|
||||
state = AgentSessionStateValues.FAILED,
|
||||
latestResult = null,
|
||||
latestError = "Permission denied.",
|
||||
),
|
||||
ParentSessionChildSummary(
|
||||
sessionId = "child-2",
|
||||
targetPackage = "com.android.settings",
|
||||
state = AgentSessionStateValues.COMPLETED,
|
||||
latestResult = "Completed task.",
|
||||
targetPresentation = AgentTargetPresentationValues.DETACHED_HIDDEN,
|
||||
requiredFinalPresentationPolicy = SessionFinalPresentationPolicy.DETACHED_SHOWN,
|
||||
latestResult = "Started the stopwatch.",
|
||||
latestError = null,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
assertEquals(AgentSessionStateValues.FAILED, rollup.state)
|
||||
assertNull(rollup.resultMessage)
|
||||
assertEquals(emptyList<String>(), rollup.sessionsToAttach)
|
||||
assertEquals(
|
||||
"Delegated session failed; com.android.deskclock: Permission denied.",
|
||||
"Delegated session completed without the required final presentation; com.android.deskclock: required DETACHED_SHOWN, actual DETACHED_HIDDEN",
|
||||
rollup.errorMessage,
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rollupCompletesWhenRequiredPresentationMatches() {
|
||||
val rollup = AgentParentSessionAggregator.rollup(
|
||||
listOf(
|
||||
ParentSessionChildSummary(
|
||||
sessionId = "child-1",
|
||||
targetPackage = "com.android.deskclock",
|
||||
state = AgentSessionStateValues.COMPLETED,
|
||||
targetPresentation = AgentTargetPresentationValues.ATTACHED,
|
||||
requiredFinalPresentationPolicy = SessionFinalPresentationPolicy.ATTACHED,
|
||||
latestResult = "Started the stopwatch.",
|
||||
latestError = null,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
assertEquals(AgentSessionStateValues.COMPLETED, rollup.state)
|
||||
assertEquals(emptyList<String>(), rollup.sessionsToAttach)
|
||||
assertEquals(
|
||||
"Completed delegated session; com.android.deskclock: Started the stopwatch.",
|
||||
rollup.resultMessage,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,8 @@ class AgentTaskPlannerTest {
|
||||
"targets": [
|
||||
{
|
||||
"packageName": "com.android.deskclock",
|
||||
"objective": "Start the requested timer in Clock."
|
||||
"objective": "Start the requested timer in Clock.",
|
||||
"finalPresentationPolicy": "ATTACHED"
|
||||
}
|
||||
],
|
||||
"reason": "DeskClock is the installed timer handler.",
|
||||
@@ -30,6 +31,10 @@ class AgentTaskPlannerTest {
|
||||
assertEquals(1, request.plan.targets.size)
|
||||
assertEquals("com.android.deskclock", request.plan.targets.single().packageName)
|
||||
assertEquals("Start the requested timer in Clock.", request.plan.targets.single().objective)
|
||||
assertEquals(
|
||||
SessionFinalPresentationPolicy.ATTACHED,
|
||||
request.plan.targets.single().finalPresentationPolicy,
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -52,6 +57,10 @@ class AgentTaskPlannerTest {
|
||||
)
|
||||
|
||||
assertEquals("Start a 5-minute timer.", request.plan.targets.single().objective)
|
||||
assertEquals(
|
||||
SessionFinalPresentationPolicy.AGENT_CHOICE,
|
||||
request.plan.targets.single().finalPresentationPolicy,
|
||||
)
|
||||
assertEquals(true, request.allowDetachedMode)
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ This Codex runtime is operating on an Android device through the Agent Platform.
|
||||
|
||||
- Detached launch, shown-detached, and attached are different states.
|
||||
- `targetDetached=true` means the target is still detached even if it is visible in a detached or mirrored presentation.
|
||||
- If the delegated objective specifies a required final target presentation such as `ATTACHED`, `DETACHED_HIDDEN`, or `DETACHED_SHOWN`, treat that as a hard completion requirement and do not claim success until the framework session matches it.
|
||||
- If the task says the app should be visible to the user, do not claim success until the target is attached unless the task explicitly allows detached presentation.
|
||||
- If the user asks to show an activity on the screen, the Genie must explicitly make its display visible. Launching hidden or leaving the target detached is not enough.
|
||||
- Treat framework session state as the source of truth for presentation state.
|
||||
|
||||
@@ -564,6 +564,7 @@ class CodexAppServerHost(
|
||||
The Genie may request detached target launch through the framework callback, and after that it may use supported self-targeted shell commands to drive the target app.
|
||||
If a direct intent launch does not fully complete the task, use detached-target tools to show or inspect the target, then continue with supported shell input and inspection surfaces.
|
||||
Use Android dynamic tools only for framework-only detached target operations that do not have a working shell equivalent in the paired app sandbox.
|
||||
The delegated objective may include a required final target presentation such as ATTACHED, DETACHED_HIDDEN, or DETACHED_SHOWN. Treat that as a hard completion requirement and do not report success until the framework session actually matches it.
|
||||
If you need clarification or a decision from the supervising Agent, call request_user_input with concise free-form question text.
|
||||
Do not use hidden control protocols.
|
||||
Finish with a normal assistant message describing what you accomplished or what blocked you.
|
||||
|
||||
@@ -59,6 +59,14 @@ The current repo now contains these implementation slices:
|
||||
runtime before falling back to notification/UI escalation, and now submits
|
||||
those answers through the same framework-session bridge instead of a separate
|
||||
Kotlin-only path.
|
||||
- The Agent now records an explicit per-child final presentation policy
|
||||
(`ATTACHED`, `DETACHED_HIDDEN`, `DETACHED_SHOWN`, or `AGENT_CHOICE`) and
|
||||
uses the framework-authoritative `AgentSessionInfo.getTargetPresentation()`
|
||||
state to verify whether a completed child actually satisfied it.
|
||||
- Parent roll-up now uses the new presentation state. If a child was required
|
||||
to finish `ATTACHED` but completes detached, the Agent requests `attachTarget`
|
||||
before rolling the parent up to success. Detached shown/hidden mismatches are
|
||||
treated as real completion errors instead of silent success.
|
||||
- Runtime testing on the emulator showed that direct cross-app `bindService`
|
||||
and raw local-socket access from the live Genie runtime are not a stable
|
||||
contract because the runtime executes under the paired target sandbox
|
||||
@@ -151,6 +159,9 @@ the Android Agent/Genie flow.
|
||||
- Dedicated framework-session bridge tools for direct Genie-session launch and question resolution
|
||||
- Framework session inspection UI in the Agent app
|
||||
- Question answering and detached-target attach controls
|
||||
- Explicit per-child final target presentation policy in planning/session
|
||||
launch, backed by framework-authoritative presentation state in session
|
||||
observation and diagnostics
|
||||
- Framework session bridge request handling in `AgentSessionBridgeServer`
|
||||
- Framework session bridge request issuance in `CodexGenieService`
|
||||
- Agent-hosted runtime metadata for Genie bootstrap
|
||||
|
||||
Reference in New Issue
Block a user