feat: migrate to new constraint-based loading strategy (#8251)

This is a significant change to how layers of configuration are applied.
In particular, the `ConfigLayerStack` now has two important fields:

- `layers: Vec<ConfigLayerEntry>`
- `requirements: ConfigRequirements`

We merge `TomlValue`s across the layers, but they are subject to
`ConfigRequirements` before creating a `Config`.

How I would review this PR:

- start with `codex-rs/app-server-protocol/src/protocol/v2.rs` and note
the new variants added to the `ConfigLayerSource` enum:
`LegacyManagedConfigTomlFromFile` and `LegacyManagedConfigTomlFromMdm`
- note that `ConfigLayerSource` now has a `precedence()` method and
implements `PartialOrd`
- `codex-rs/core/src/config_loader/layer_io.rs` is responsible for
loading "admin" preferences from `/etc/codex/managed_config.toml` and
MDM. Because `/etc/codex/managed_config.toml` is now deprecated in favor
of `/etc/codex/requirements.toml` and `/etc/codex/config.toml`, we now
include some extra information on the `LoadedConfigLayers` returned in
`layer_io.rs`.
- `codex-rs/core/src/config_loader/mod.rs` has major changes to
`load_config_layers_state()`, which is what produces `ConfigLayerStack`.
The docstring has the new specification and describes the various layers
that will be loaded and the precedence order.
- It uses the information from `LoaderOverrides` "twice," both in the
spirit of legacy support:
- We use one instances to derive an instance of `ConfigRequirements`.
Currently, the only field in `managed_config.toml` that contributes to
`ConfigRequirements` is `approval_policy`. This PR introduces
`Constrained::allow_only()` to support this.
- We use a clone of `LoaderOverrides` to derive
`ConfigLayerSource::LegacyManagedConfigTomlFromFile` and
`ConfigLayerSource::LegacyManagedConfigTomlFromMdm` layers, as
appropriate. As before, this ends up being a "best effort" at enterprise
controls, but is enforcement is not guaranteed like it is for
`ConfigRequirements`.
- Now we only create a "user" layer if `$CODEX_HOME/config.toml` exists.
(Previously, a user layer was always created for `ConfigLayerStack`.)
- Similarly, we only add a "session flags" layer if there are CLI
overrides.
- `config_loader/state.rs` contains the updated implementation for
`ConfigLayerStack`. Note the public API is largely the same as before,
but the implementation is quite different. We leverage the fact that
`ConfigLayerSource` is now `PartialOrd` to ensure layers are in the
correct order.
- A `Config` constructed via `ConfigBuilder.build()` will use
`load_config_layers_state()` to create the `ConfigLayerStack` and use
the associated `ConfigRequirements` when constructing the `Config`
object.
- That said, a `Config` constructed via
`Config::load_from_base_config_with_overrides()` does _not_ yet use
`ConfigBuilder`, so it creates a `ConfigRequirements::default()` instead
of loading a proper `ConfigRequirements`. I will fix this in a
subsequent PR.

Then the following files are mostly test changes:

```
codex-rs/app-server/tests/suite/v2/config_rpc.rs
codex-rs/core/src/config/service.rs
codex-rs/core/src/config_loader/tests.rs
```

Again, because we do not always include "user" and "session flags"
layers when the contents are empty, `ConfigLayerStack` sometimes has
fewer layers than before (and the precedence order changed slightly),
which is the main reason integration tests changed.

codex-rs/core/src/config_loader/state.rs

@@ -1,9 +1,12 @@
+use crate::config_loader::ConfigRequirements;
+
 use super::fingerprint::record_origins;
 use super::fingerprint::version_for_toml;
 use super::merge::merge_toml_values;
 use codex_app_server_protocol::ConfigLayer;
 use codex_app_server_protocol::ConfigLayerMetadata;
 use codex_app_server_protocol::ConfigLayerSource;
+use codex_utils_absolute_path::AbsolutePathBuf;
 use serde_json::Value as JsonValue;
 use std::collections::HashMap;
 use std::path::PathBuf;
@@ -51,30 +54,90 @@ impl ConfigLayerEntry {
 
 #[derive(Debug, Clone)]
 pub struct ConfigLayerStack {
-    pub user: ConfigLayerEntry,
-    pub session_flags: ConfigLayerEntry,
-    pub system: Option<ConfigLayerEntry>,
-    pub mdm: Option<ConfigLayerEntry>,
+    /// Layers are listed from lowest precedence (base) to highest (top), so
+    /// later entries in the Vec override earlier ones.
+    layers: Vec<ConfigLayerEntry>,
+
+    /// Index into [layers] of the user config layer, if any.
+    user_layer_index: Option<usize>,
+
+    /// Constraints that must be enforced when deriving a [Config] from the
+    /// layers.
+    requirements: ConfigRequirements,
 }
 
 impl ConfigLayerStack {
-    pub fn with_user_config(&self, user_config: TomlValue) -> Self {
-        Self {
-            user: ConfigLayerEntry::new(self.user.name.clone(), user_config),
-            session_flags: self.session_flags.clone(),
-            system: self.system.clone(),
-            mdm: self.mdm.clone(),
+    pub fn new(
+        layers: Vec<ConfigLayerEntry>,
+        requirements: ConfigRequirements,
+    ) -> std::io::Result<Self> {
+        let user_layer_index = verify_layer_ordering(&layers)?;
+        Ok(Self {
+            layers,
+            user_layer_index,
+            requirements,
+        })
+    }
+
+    /// Returns the user config layer, if any.
+    pub fn get_user_layer(&self) -> Option<&ConfigLayerEntry> {
+        self.user_layer_index
+            .and_then(|index| self.layers.get(index))
+    }
+
+    pub fn requirements(&self) -> &ConfigRequirements {
+        &self.requirements
+    }
+
+    /// Creates a new [ConfigLayerStack] using the specified values to inject a
+    /// "user layer" into the stack. If such a layer already exists, it is
+    /// replaced; otherwise, it is inserted into the stack at the appropriate
+    /// position based on precedence rules.
+    pub fn with_user_config(&self, config_toml: &AbsolutePathBuf, user_config: TomlValue) -> Self {
+        let user_layer = ConfigLayerEntry::new(
+            ConfigLayerSource::User {
+                file: config_toml.clone(),
+            },
+            user_config,
+        );
+
+        let mut layers = self.layers.clone();
+        match self.user_layer_index {
+            Some(index) => {
+                layers[index] = user_layer;
+                Self {
+                    layers,
+                    user_layer_index: self.user_layer_index,
+                    requirements: self.requirements.clone(),
+                }
+            }
+            None => {
+                let user_layer_index = match layers
+                    .iter()
+                    .position(|layer| layer.name.precedence() > user_layer.name.precedence())
+                {
+                    Some(index) => {
+                        layers.insert(index, user_layer);
+                        index
+                    }
+                    None => {
+                        layers.push(user_layer);
+                        layers.len() - 1
+                    }
+                };
+                Self {
+                    layers,
+                    user_layer_index: Some(user_layer_index),
+                    requirements: self.requirements.clone(),
+                }
+            }
         }
     }
 
     pub fn effective_config(&self) -> TomlValue {
-        let mut merged = self.user.config.clone();
-        merge_toml_values(&mut merged, &self.session_flags.config);
-        if let Some(system) = &self.system {
-            merge_toml_values(&mut merged, &system.config);
-        }
-        if let Some(mdm) = &self.mdm {
-            merge_toml_values(&mut merged, &mdm.config);
+        let mut merged = TomlValue::Table(toml::map::Map::new());
+        for layer in &self.layers {
+            merge_toml_values(&mut merged, &layer.config);
         }
         merged
     }
@@ -83,38 +146,42 @@ impl ConfigLayerStack {
         let mut origins = HashMap::new();
         let mut path = Vec::new();
 
-        record_origins(
-            &self.user.config,
-            &self.user.metadata(),
-            &mut path,
-            &mut origins,
-        );
-        record_origins(
-            &self.session_flags.config,
-            &self.session_flags.metadata(),
-            &mut path,
-            &mut origins,
-        );
-        if let Some(system) = &self.system {
-            record_origins(&system.config, &system.metadata(), &mut path, &mut origins);
-        }
-        if let Some(mdm) = &self.mdm {
-            record_origins(&mdm.config, &mdm.metadata(), &mut path, &mut origins);
+        for layer in &self.layers {
+            record_origins(&layer.config, &layer.metadata(), &mut path, &mut origins);
         }
 
         origins
     }
 
-    pub fn layers_high_to_low(&self) -> Vec<ConfigLayer> {
-        let mut layers = Vec::new();
-        if let Some(mdm) = &self.mdm {
-            layers.push(mdm.as_layer());
-        }
-        if let Some(system) = &self.system {
-            layers.push(system.as_layer());
-        }
-        layers.push(self.session_flags.as_layer());
-        layers.push(self.user.as_layer());
-        layers
+    /// Returns the highest-precedence to lowest-precedence layers, so
+    /// `ConfigLayerSource::SessionFlags` would be first, if present.
+    pub fn layers_high_to_low(&self) -> Vec<&ConfigLayerEntry> {
+        self.layers.iter().rev().collect()
     }
 }
+
+/// Ensures precedence ordering of config layers is correct. Returns the index
+/// of the user config layer, if any (at most one should exist).
+fn verify_layer_ordering(layers: &[ConfigLayerEntry]) -> std::io::Result<Option<usize>> {
+    if !layers.iter().map(|layer| &layer.name).is_sorted() {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidData,
+            "config layers are not in correct precedence order",
+        ));
+    }
+
+    let mut user_layer_index: Option<usize> = None;
+    for (index, layer) in layers.iter().enumerate() {
+        if matches!(layer.name, ConfigLayerSource::User { .. }) {
+            if user_layer_index.is_some() {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidData,
+                    "multiple user config layers found",
+                ));
+            }
+            user_layer_index = Some(index);
+        }
+    }
+
+    Ok(user_layer_index)
+}