Compare commits

...

2 Commits

Author SHA1 Message Date
Felipe Coury
727e6a775e refactor(tui): simplify table stream holdback 2026-04-28 13:19:57 -03:00
Felipe Coury
5dcaeda334 feat(tui): add streaming markdown table support 2026-04-28 13:10:04 -03:00
17 changed files with 4175 additions and 237 deletions

View File

@@ -181,6 +181,7 @@ use tokio::sync::mpsc::unbounded_channel;
use tokio::task::JoinHandle;
use toml::Value as TomlValue;
use uuid::Uuid;
mod agent_message_consolidation;
mod agent_navigation;
mod app_server_adapter;
pub(crate) mod app_server_requests;

View File

@@ -0,0 +1,75 @@
use std::path::PathBuf;
use std::sync::Arc;
use color_eyre::eyre::Result;
use super::App;
use super::resize_reflow::trailing_run_start;
use crate::app_event::ConsolidationScrollbackReflow;
use crate::history_cell;
use crate::pager_overlay::Overlay;
use crate::tui;
impl App {
pub(super) fn handle_consolidate_agent_message(
&mut self,
tui: &mut tui::Tui,
source: String,
cwd: PathBuf,
scrollback_reflow: ConsolidationScrollbackReflow,
) -> Result<()> {
// Walk backward to find the contiguous run of streaming AgentMessageCells that
// belong to the just-finalized stream.
let end = self.transcript_cells.len();
tracing::debug!(
"ConsolidateAgentMessage: transcript_cells.len()={end}, source_len={}",
source.len()
);
let start = trailing_run_start::<history_cell::AgentMessageCell>(&self.transcript_cells);
let consolidated: Arc<dyn crate::history_cell::HistoryCell> =
Arc::new(history_cell::AgentMarkdownCell::new(source, &cwd));
if start < end {
tracing::debug!(
"ConsolidateAgentMessage: replacing cells [{start}..{end}] with AgentMarkdownCell"
);
self.transcript_cells
.splice(start..end, std::iter::once(consolidated.clone()));
if let Some(Overlay::Transcript(t)) = &mut self.overlay {
t.consolidate_cells(start..end, consolidated.clone());
tui.frame_requester().schedule_frame();
}
self.finish_agent_message_consolidation(tui, scrollback_reflow)?;
} else {
tracing::debug!(
"ConsolidateAgentMessage: inserting AgentMarkdownCell(start={start}, end={end})",
);
self.transcript_cells.push(consolidated.clone());
if let Some(Overlay::Transcript(t)) = &mut self.overlay {
t.insert_cell(consolidated);
tui.frame_requester().schedule_frame();
}
self.finish_agent_message_consolidation(tui, scrollback_reflow)?;
}
Ok(())
}
fn finish_agent_message_consolidation(
&mut self,
tui: &mut tui::Tui,
scrollback_reflow: ConsolidationScrollbackReflow,
) -> Result<()> {
match scrollback_reflow {
ConsolidationScrollbackReflow::IfResizeReflowRan => {
self.maybe_finish_stream_reflow(tui)?;
}
ConsolidationScrollbackReflow::Required => {
self.finish_required_stream_reflow(tui)?;
}
}
Ok(())
}
}

View File

@@ -206,29 +206,12 @@ impl App {
AppEvent::EndInitialHistoryReplayBuffer => {
self.finish_initial_history_replay_buffer(tui);
}
AppEvent::ConsolidateAgentMessage { source, cwd } => {
if !self.terminal_resize_reflow_enabled() {
self.transcript_reflow.clear();
return Ok(AppRunControl::Continue);
}
let end = self.transcript_cells.len();
let start =
trailing_run_start::<history_cell::AgentMessageCell>(&self.transcript_cells);
if start < end {
let consolidated: Arc<dyn HistoryCell> =
Arc::new(history_cell::AgentMarkdownCell::new(source, &cwd));
self.transcript_cells
.splice(start..end, std::iter::once(consolidated.clone()));
if let Some(Overlay::Transcript(t)) = &mut self.overlay {
t.consolidate_cells(start..end, consolidated.clone());
tui.frame_requester().schedule_frame();
}
self.maybe_finish_stream_reflow(tui)?;
} else {
self.maybe_finish_stream_reflow(tui)?;
}
AppEvent::ConsolidateAgentMessage {
source,
cwd,
scrollback_reflow,
} => {
self.handle_consolidate_agent_message(tui, source, cwd, scrollback_reflow)?;
}
AppEvent::ConsolidateProposedPlan(source) => {
if !self.terminal_resize_reflow_enabled() {

View File

@@ -77,6 +77,12 @@ impl RealtimeAudioDeviceKind {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ConsolidationScrollbackReflow {
IfResizeReflowRan,
Required,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(not(target_os = "windows"), allow(dead_code))]
pub(crate) enum WindowsSandboxEnableMode {
@@ -413,9 +419,12 @@ pub(crate) enum AppEvent {
/// finalization. The `App` handler walks backward through `transcript_cells`
/// to find the `AgentMessageCell` run and splices in the consolidated cell.
/// The `cwd` keeps local file-link display stable across the final re-render.
/// `scrollback_reflow` lets table-tail finalization force the already-emitted
/// terminal scrollback to be rebuilt from the consolidated source-backed cell.
ConsolidateAgentMessage {
source: String,
cwd: PathBuf,
scrollback_reflow: ConsolidationScrollbackReflow,
},
/// Replace the contiguous run of streaming `ProposedPlanStreamCell`s at the

View File

@@ -2026,8 +2026,17 @@ impl ChatWidget {
fn flush_answer_stream_with_separator(&mut self) {
let had_stream_controller = self.stream_controller.is_some();
if let Some(mut controller) = self.stream_controller.take() {
let scrollback_reflow = if controller.has_live_tail() {
crate::app_event::ConsolidationScrollbackReflow::Required
} else {
crate::app_event::ConsolidationScrollbackReflow::IfResizeReflowRan
};
self.clear_active_stream_tail();
let (cell, source) = controller.finalize();
if let Some(cell) = cell {
if scrollback_reflow
== crate::app_event::ConsolidationScrollbackReflow::IfResizeReflowRan
&& let Some(cell) = cell
{
self.add_boxed_history(cell);
}
// Consolidate the run of streaming AgentMessageCells into a single AgentMarkdownCell
@@ -2036,6 +2045,7 @@ impl ChatWidget {
self.app_event_tx.send(AppEvent::ConsolidateAgentMessage {
source,
cwd: self.config.cwd.to_path_buf(),
scrollback_reflow,
});
}
}
@@ -2610,11 +2620,10 @@ impl ChatWidget {
self.plan_delta_buffer.clear();
}
self.plan_delta_buffer.push_str(&delta);
// Before streaming plan content, flush any active exec cell group.
self.flush_unified_exec_wait_streak();
self.flush_active_cell();
if self.plan_stream_controller.is_none() {
// Before starting a plan stream, flush any active exec cell group.
self.flush_unified_exec_wait_streak();
self.flush_active_cell();
self.plan_stream_controller = Some(PlanStreamController::new(
self.current_stream_width(/*reserved_cols*/ 4),
&self.config.cwd,
@@ -2626,6 +2635,7 @@ impl ChatWidget {
self.app_event_tx.send(AppEvent::StartCommitAnimation);
self.run_catch_up_commit_tick();
}
self.sync_active_stream_tail();
self.request_redraw();
}
@@ -2648,7 +2658,14 @@ impl ChatWidget {
self.saw_plan_item_this_turn = true;
let (finalized_streamed_cell, consolidated_plan_source) =
if let Some(mut controller) = self.plan_stream_controller.take() {
controller.finalize()
let had_live_tail = controller.has_live_tail();
self.clear_active_stream_tail();
let (cell, source) = controller.finalize();
if had_live_tail {
(None, source)
} else {
(cell, source)
}
} else {
(None, None)
};
@@ -2783,8 +2800,10 @@ impl ChatWidget {
// If a stream is currently active, finalize it.
self.flush_answer_stream_with_separator();
if let Some(mut controller) = self.plan_stream_controller.take() {
let had_live_tail = controller.has_live_tail();
self.clear_active_stream_tail();
let (cell, source) = controller.finalize();
if let Some(cell) = cell {
if !had_live_tail && let Some(cell) = cell {
self.add_boxed_history(cell);
}
if let Some(source) = source {
@@ -3280,6 +3299,9 @@ impl ChatWidget {
/// This does not clear MCP startup tracking, because MCP startup can overlap with turn cleanup
/// and should continue to drive the bottom-pane running indicator while it is in progress.
fn finalize_turn(&mut self) {
// Drop preview-only stream tail content on any termination path before
// failed-cell finalization, so transient tail cells are never persisted.
self.clear_active_stream_tail();
// Ensure any spinner is replaced by a red ✗ and flushed into history.
self.finalize_active_cell_as_failed();
// Reset running state and clear streaming buffers.
@@ -4982,6 +5004,7 @@ impl ChatWidget {
self.bottom_pane.hide_status_indicator();
self.add_boxed_history(cell);
}
self.sync_active_stream_tail();
if outcome.has_controller && outcome.all_idle {
self.maybe_restore_status_indicator_after_stream_idle();
@@ -5026,11 +5049,10 @@ impl ChatWidget {
#[inline]
fn handle_streaming_delta(&mut self, delta: String) {
// Before streaming agent content, flush any active exec cell group.
self.flush_unified_exec_wait_streak();
self.flush_active_cell();
if self.stream_controller.is_none() {
// Before starting an agent stream, flush any active exec cell group.
self.flush_unified_exec_wait_streak();
self.flush_active_cell();
// If the previous turn inserted non-stream history (exec output, patch status, MCP
// calls), render a separator before starting the next streamed assistant message.
if self.needs_final_message_separator && self.had_work_activity {
@@ -5060,6 +5082,7 @@ impl ChatWidget {
self.app_event_tx.send(AppEvent::StartCommitAnimation);
self.run_catch_up_commit_tick();
}
self.sync_active_stream_tail();
self.request_redraw();
}
@@ -6116,12 +6139,69 @@ impl ChatWidget {
if !keep_placeholder_header_active && !cell.display_lines(u16::MAX).is_empty() {
// Only break exec grouping if the cell renders visible lines.
self.flush_active_cell();
if !self.has_active_stream_tail() {
self.flush_active_cell();
}
self.needs_final_message_separator = true;
}
self.app_event_tx.send(AppEvent::InsertHistoryCell(cell));
}
fn active_cell_is_stream_tail(&self) -> bool {
self.active_cell.as_ref().is_some_and(|cell| {
cell.as_any().is::<history_cell::StreamingAgentTailCell>()
|| cell.as_any().is::<history_cell::StreamingPlanTailCell>()
})
}
fn has_active_stream_tail(&self) -> bool {
(self.stream_controller.is_some() || self.plan_stream_controller.is_some())
&& self.active_cell_is_stream_tail()
}
fn sync_active_stream_tail(&mut self) {
if let Some(controller) = self.stream_controller.as_ref() {
let tail_lines = controller.current_tail_lines();
if tail_lines.is_empty() {
self.clear_active_stream_tail();
return;
}
self.bottom_pane.hide_status_indicator();
self.active_cell = Some(Box::new(history_cell::StreamingAgentTailCell::new(
tail_lines,
controller.tail_starts_stream(),
)));
self.bump_active_cell_revision();
return;
}
if let Some(controller) = self.plan_stream_controller.as_ref() {
let tail_lines = controller.current_tail_display_lines();
if tail_lines.is_empty() {
self.clear_active_stream_tail();
return;
}
self.bottom_pane.hide_status_indicator();
self.active_cell = Some(Box::new(history_cell::StreamingPlanTailCell::new(
tail_lines,
!controller.tail_starts_stream(),
)));
self.bump_active_cell_revision();
return;
}
self.clear_active_stream_tail();
}
fn clear_active_stream_tail(&mut self) {
if self.active_cell_is_stream_tail() {
self.active_cell = None;
self.bump_active_cell_revision();
}
}
fn queue_user_message(&mut self, user_message: UserMessage) {
self.queue_user_message_with_options(user_message, QueuedInputAction::Plain);
}
@@ -11542,6 +11622,7 @@ impl ChatWidget {
if let Some(controller) = self.plan_stream_controller.as_mut() {
controller.set_width(plan_stream_width);
}
self.sync_active_stream_tail();
if !had_rendered_width {
self.request_redraw();
}
@@ -11712,6 +11793,7 @@ impl ChatWidget {
if let Some(controller) = self.plan_stream_controller.as_mut() {
controller.clear_queue();
}
self.clear_active_stream_tail();
self.request_redraw();
}
}

View File

@@ -153,6 +153,133 @@ async fn turn_started_uses_runtime_context_window_before_first_token_count() {
"expected /status to avoid raw config context window, got: {context_line}"
);
}
#[tokio::test]
async fn flush_answer_stream_keeps_default_reflow_for_plain_text_tail() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
let cwd = chat.config.cwd.to_path_buf();
let mut controller =
crate::streaming::controller::StreamController::new(Some(80), cwd.as_path());
assert!(controller.push("plain response line\n"));
chat.stream_controller = Some(controller);
while rx.try_recv().is_ok() {}
chat.flush_answer_stream_with_separator();
let mut saw_consolidate = false;
let mut saw_insert_history = false;
while let Ok(event) = rx.try_recv() {
match event {
AppEvent::InsertHistoryCell(_) => saw_insert_history = true,
AppEvent::ConsolidateAgentMessage {
scrollback_reflow, ..
} => {
saw_consolidate = true;
assert_eq!(
scrollback_reflow,
crate::app_event::ConsolidationScrollbackReflow::IfResizeReflowRan
);
}
_ => {}
}
}
assert!(
saw_consolidate,
"expected stream finalization to consolidate"
);
assert!(
saw_insert_history,
"plain text should still insert history before consolidation"
);
}
#[tokio::test]
async fn flush_answer_stream_requests_scrollback_reflow_for_live_table_tail() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
let cwd = chat.config.cwd.to_path_buf();
let mut controller =
crate::streaming::controller::StreamController::new(Some(80), cwd.as_path());
controller.push("| Name | Notes |\n");
controller.push("| --- | --- |\n");
controller.push("| alpha | tail held until final table render |\n");
assert!(
controller.has_live_tail(),
"expected table holdback to leave a live tail for this regression",
);
chat.stream_controller = Some(controller);
while rx.try_recv().is_ok() {}
chat.flush_answer_stream_with_separator();
let mut saw_consolidate = false;
let mut saw_insert_history = false;
while let Ok(event) = rx.try_recv() {
match event {
AppEvent::InsertHistoryCell(_) => saw_insert_history = true,
AppEvent::ConsolidateAgentMessage {
scrollback_reflow, ..
} => {
saw_consolidate = true;
assert_eq!(
scrollback_reflow,
crate::app_event::ConsolidationScrollbackReflow::Required
);
}
_ => {}
}
}
assert!(
saw_consolidate,
"expected stream finalization to consolidate"
);
assert!(
!saw_insert_history,
"live table tail should not be inserted before canonical reflow"
);
}
#[tokio::test]
async fn completed_plan_table_tail_skips_provisional_history_insert() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
let cwd = chat.config.cwd.to_path_buf();
let mut controller =
crate::streaming::controller::PlanStreamController::new(Some(80), cwd.as_path());
controller.push("| Step | Owner |\n");
controller.push("| --- | --- |\n");
controller.push("| Verify | Codex |\n");
assert!(
controller.has_live_tail(),
"expected plan table holdback to leave a live tail",
);
chat.plan_stream_controller = Some(controller);
chat.plan_delta_buffer = "| Step | Owner |\n| --- | --- |\n| Verify | Codex |\n".to_string();
while rx.try_recv().is_ok() {}
chat.on_plan_item_completed(String::new());
let mut saw_source_backed_plan = false;
let mut saw_stream_plan = false;
while let Ok(event) = rx.try_recv() {
if let AppEvent::InsertHistoryCell(cell) = event {
saw_source_backed_plan |= cell.as_any().is::<history_cell::ProposedPlanCell>();
saw_stream_plan |= cell.as_any().is::<history_cell::ProposedPlanStreamCell>();
}
}
assert!(saw_source_backed_plan, "expected source-backed plan insert");
assert!(
!saw_stream_plan,
"live plan table tail should not be inserted provisionally"
);
}
#[tokio::test]
async fn helpers_are_available_and_do_not_panic() {
let (tx_raw, _rx) = unbounded_channel::<AppEvent>();

View File

@@ -493,7 +493,8 @@ impl HistoryCell for AgentMessageCell {
/// After a stream finalizes, the `ConsolidateAgentMessage` handler in `App`
/// replaces the contiguous run of `AgentMessageCell`s with a single
/// `AgentMarkdownCell`. On terminal resize, `display_lines(width)` re-renders
/// from source via `append_markdown`.
/// from source via `append_markdown_agent`, producing correctly-sized tables
/// with box-drawing borders.
///
/// The cell snapshots `cwd` at construction so local file-link display remains aligned with the
/// session that produced the message. Reusing the current process cwd during reflow would make old
@@ -529,7 +530,7 @@ impl HistoryCell for AgentMarkdownCell {
let mut lines: Vec<Line<'static>> = Vec::new();
// Re-render markdown from source at the current width. Reserve 2 columns for the "• " /
// " " prefix prepended below.
crate::markdown::append_markdown(
crate::markdown::append_markdown_agent_with_cwd(
&self.markdown_source,
Some(wrap_width),
Some(self.cwd.as_path()),
@@ -539,6 +540,76 @@ impl HistoryCell for AgentMarkdownCell {
}
}
/// Transient active-cell representation of the mutable tail of an agent stream.
///
/// During streaming, lines that have not yet been committed to scrollback because they belong to
/// an in-progress table are displayed via this cell in the `active_cell` slot. It is replaced on
/// every delta and cleared when the stream finalizes.
#[derive(Debug)]
pub(crate) struct StreamingAgentTailCell {
lines: Vec<Line<'static>>,
is_first_line: bool,
}
impl StreamingAgentTailCell {
pub(crate) fn new(lines: Vec<Line<'static>>, is_first_line: bool) -> Self {
Self {
lines,
is_first_line,
}
}
}
impl HistoryCell for StreamingAgentTailCell {
fn display_lines(&self, _width: u16) -> Vec<Line<'static>> {
// Tail lines are already rendered at the controller's current stream width.
// Re-wrapping them here can split table borders and produce malformed in-flight rows.
prefix_lines(
self.lines.clone(),
if self.is_first_line {
"".dim()
} else {
" ".into()
},
" ".into(),
)
}
fn is_stream_continuation(&self) -> bool {
!self.is_first_line
}
}
/// Transient active-cell representation of the mutable tail of a proposed-plan stream.
///
/// The controller prepares the full styled plan lines because plan tails need the same header,
/// padding, and background treatment as committed `ProposedPlanStreamCell`s while remaining
/// preview-only during streaming.
#[derive(Debug)]
pub(crate) struct StreamingPlanTailCell {
lines: Vec<Line<'static>>,
is_stream_continuation: bool,
}
impl StreamingPlanTailCell {
pub(crate) fn new(lines: Vec<Line<'static>>, is_stream_continuation: bool) -> Self {
Self {
lines,
is_stream_continuation,
}
}
}
impl HistoryCell for StreamingPlanTailCell {
fn display_lines(&self, _width: u16) -> Vec<Line<'static>> {
self.lines.clone()
}
fn is_stream_continuation(&self) -> bool {
self.is_stream_continuation
}
}
#[derive(Debug)]
pub(crate) struct PlainHistoryCell {
lines: Vec<Line<'static>>,

View File

@@ -240,6 +240,7 @@ mod voice {
mod wrapping;
mod table_detect;
#[cfg(test)]
pub(crate) mod test_backend;
#[cfg(test)]

View File

@@ -1,7 +1,32 @@
//! Markdown-to-ratatui rendering entry points.
//!
//! This module provides the public API surface that the rest of the TUI uses
//! to turn markdown source into `Vec<Line<'static>>`. Two variants exist:
//!
//! - [`append_markdown`] -- general-purpose, used for plan blocks and history
//! cells that already hold pre-processed markdown (no fence unwrapping).
//! - [`append_markdown_agent`] -- for agent responses. Runs
//! [`unwrap_markdown_fences`] first so that `` ```md ``/`` ```markdown ``
//! fences containing tables are stripped and `pulldown-cmark` sees raw
//! table syntax instead of fenced code.
//!
//! ## Why fence unwrapping exists
//!
//! LLM agents frequently wrap tables in `` ```markdown `` fences, treating
//! them as code. Without unwrapping, `pulldown-cmark` parses those lines
//! as a fenced code block and renders them as monospace code rather than a
//! structured table. The unwrapper is intentionally conservative: it
//! buffers the entire fence body before deciding, only unwraps fences whose
//! info string is `md` or `markdown` AND whose body contains a
//! header+delimiter pair, and degrades gracefully on unclosed fences.
use ratatui::text::Line;
use std::borrow::Cow;
use std::ops::Range;
use std::path::Path;
/// Render markdown into `lines` while resolving local file-link display relative to `cwd`.
use crate::table_detect;
/// Render markdown source to styled ratatui lines and append them to `lines`.
///
/// Callers that already know the session working directory should pass it here so streamed and
/// non-streamed rendering show the same relative path text even if the process cwd differs.
@@ -19,6 +44,253 @@ pub(crate) fn append_markdown(
crate::render::line_utils::push_owned_lines(&rendered.lines, lines);
}
/// Render an agent message to styled ratatui lines.
///
/// Before rendering, the source is passed through [`unwrap_markdown_fences`] so that tables
/// wrapped in `` ```md `` fences are rendered as native tables rather than code blocks.
/// Non-markdown fences (e.g. `rust`, `sh`) are left
/// intact.
#[cfg(test)]
pub(crate) fn append_markdown_agent(
markdown_source: &str,
width: Option<usize>,
lines: &mut Vec<Line<'static>>,
) {
append_markdown_agent_with_cwd(markdown_source, width, /*cwd*/ None, lines);
}
/// Render an agent message while resolving local file links relative to `cwd`.
pub(crate) fn append_markdown_agent_with_cwd(
markdown_source: &str,
width: Option<usize>,
cwd: Option<&Path>,
lines: &mut Vec<Line<'static>>,
) {
let normalized = unwrap_markdown_fences(markdown_source);
let rendered =
crate::markdown_render::render_markdown_text_with_width_and_cwd(&normalized, width, cwd);
crate::render::line_utils::push_owned_lines(&rendered.lines, lines);
}
/// Strip `` ```md ``/`` ```markdown `` fences that contain tables, emitting their content as bare
/// markdown so `pulldown-cmark` parses the tables natively.
///
/// Fences whose info string is not `md` or `markdown` are passed through unchanged. Markdown
/// fences that do *not* contain a table (detected by checking for a header row + delimiter row)
/// are also passed through so that non-table markdown inside a fence still renders as a code
/// block.
///
/// The fence unwrapping is intentionally conservative: it buffers the entire fence body before
/// deciding, and an unclosed fence at end-of-input is re-emitted with its opening line so partial
/// streams degrade to code display.
fn unwrap_markdown_fences<'a>(markdown_source: &'a str) -> Cow<'a, str> {
// Zero-copy fast path: most messages contain no fences at all.
if !markdown_source.contains("```") && !markdown_source.contains("~~~") {
return Cow::Borrowed(markdown_source);
}
#[derive(Clone, Copy)]
struct Fence {
marker: u8,
len: usize,
is_blockquoted: bool,
}
// Strip a trailing newline and up to 3 leading spaces, returning the
// trimmed slice. Returns `None` when the line has 4+ leading spaces
// (which makes it an indented code line per CommonMark).
fn strip_line_indent(line: &str) -> Option<&str> {
let without_newline = line.strip_suffix('\n').unwrap_or(line);
let mut byte_idx = 0usize;
let mut column = 0usize;
for b in without_newline.as_bytes() {
match b {
b' ' => {
byte_idx += 1;
column += 1;
}
b'\t' => {
byte_idx += 1;
column += 4;
}
_ => break,
}
if column >= 4 {
return None;
}
}
Some(&without_newline[byte_idx..])
}
// Parse an opening fence line, returning the fence metadata and whether
// the fence info string indicates markdown content.
fn parse_open_fence(line: &str) -> Option<(Fence, bool)> {
let trimmed = strip_line_indent(line)?;
let is_blockquoted = trimmed.trim_start().starts_with('>');
let fence_scan_text = table_detect::strip_blockquote_prefix(trimmed);
let (marker, len) = table_detect::parse_fence_marker(fence_scan_text)?;
let is_markdown = table_detect::is_markdown_fence_info(fence_scan_text, len);
Some((
Fence {
marker: marker as u8,
len,
is_blockquoted,
},
is_markdown,
))
}
fn is_close_fence(line: &str, fence: Fence) -> bool {
let Some(trimmed) = strip_line_indent(line) else {
return false;
};
let fence_scan_text = if fence.is_blockquoted {
if !trimmed.trim_start().starts_with('>') {
return false;
}
table_detect::strip_blockquote_prefix(trimmed)
} else {
trimmed
};
if let Some((marker, len)) = table_detect::parse_fence_marker(fence_scan_text) {
marker as u8 == fence.marker
&& len >= fence.len
&& fence_scan_text[len..].trim().is_empty()
} else {
false
}
}
fn markdown_fence_contains_table(content: &str, is_blockquoted_fence: bool) -> bool {
let mut previous_line: Option<&str> = None;
for line in content.lines() {
let text = if is_blockquoted_fence {
table_detect::strip_blockquote_prefix(line)
} else {
line
};
let trimmed = text.trim();
if trimmed.is_empty() {
previous_line = None;
continue;
}
if let Some(previous) = previous_line
&& table_detect::is_table_header_line(previous)
&& !table_detect::is_table_delimiter_line(previous)
&& table_detect::is_table_delimiter_line(trimmed)
{
return true;
}
previous_line = Some(trimmed);
}
false
}
fn content_from_ranges(source: &str, ranges: &[Range<usize>]) -> String {
let total_len: usize = ranges.iter().map(ExactSizeIterator::len).sum();
let mut content = String::with_capacity(total_len);
for range in ranges {
content.push_str(&source[range.start..range.end]);
}
content
}
struct MarkdownCandidateData {
fence: Fence,
opening_range: Range<usize>,
content_ranges: Vec<Range<usize>>,
}
// Box the large variant to keep ActiveFence small (~pointer-sized).
enum ActiveFence {
Passthrough(Fence),
MarkdownCandidate(Box<MarkdownCandidateData>),
}
let mut out = String::with_capacity(markdown_source.len());
let mut active_fence: Option<ActiveFence> = None;
let mut source_offset = 0usize;
let mut push_source_range = |range: Range<usize>| {
if !range.is_empty() {
out.push_str(&markdown_source[range]);
}
};
for line in markdown_source.split_inclusive('\n') {
let line_start = source_offset;
source_offset += line.len();
let line_range = line_start..source_offset;
if let Some(active) = active_fence.take() {
match active {
ActiveFence::Passthrough(fence) => {
push_source_range(line_range);
if !is_close_fence(line, fence) {
active_fence = Some(ActiveFence::Passthrough(fence));
}
}
ActiveFence::MarkdownCandidate(mut data) => {
if is_close_fence(line, data.fence) {
if markdown_fence_contains_table(
&content_from_ranges(markdown_source, &data.content_ranges),
data.fence.is_blockquoted,
) {
for range in data.content_ranges {
push_source_range(range);
}
} else {
push_source_range(data.opening_range);
for range in data.content_ranges {
push_source_range(range);
}
push_source_range(line_range);
}
} else {
data.content_ranges.push(line_range);
active_fence = Some(ActiveFence::MarkdownCandidate(data));
}
}
}
continue;
}
if let Some((fence, is_markdown)) = parse_open_fence(line) {
if is_markdown {
active_fence = Some(ActiveFence::MarkdownCandidate(Box::new(
MarkdownCandidateData {
fence,
opening_range: line_range,
content_ranges: Vec::new(),
},
)));
} else {
push_source_range(line_range);
active_fence = Some(ActiveFence::Passthrough(fence));
}
continue;
}
push_source_range(line_range);
}
if let Some(active) = active_fence {
match active {
ActiveFence::Passthrough(_) => {}
ActiveFence::MarkdownCandidate(data) => {
push_source_range(data.opening_range);
for range in data.content_ranges {
push_source_range(range);
}
}
}
}
Cow::Owned(out)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -118,4 +390,110 @@ mod tests {
"did not expect a split into ['1.', 'Tight item']; got: {lines:?}"
);
}
#[test]
fn append_markdown_agent_unwraps_markdown_fences_for_table_rendering() {
let src = "```markdown\n| A | B |\n|---|---|\n| 1 | 2 |\n```\n";
let mut out = Vec::new();
append_markdown_agent(src, /*width*/ None, &mut out);
let rendered = lines_to_strings(&out);
assert!(rendered.iter().any(|line| line.contains("")));
assert!(rendered.iter().any(|line| line.contains("│ 1 │ 2 │")));
}
#[test]
fn append_markdown_agent_unwraps_markdown_fences_for_no_outer_table_rendering() {
let src = "```md\nCol A | Col B | Col C\n--- | --- | ---\nx | y | z\n10 | 20 | 30\n```\n";
let mut out = Vec::new();
append_markdown_agent(src, /*width*/ None, &mut out);
let rendered = lines_to_strings(&out);
assert!(rendered.iter().any(|line| line.contains("")));
assert!(
rendered
.iter()
.any(|line| line.contains("│ Col A │ Col B │ Col C │"))
);
assert!(
!rendered
.iter()
.any(|line| line.trim() == "Col A | Col B | Col C")
);
}
#[test]
fn append_markdown_agent_unwraps_markdown_fences_for_two_column_no_outer_table() {
let src = "```md\nA | B\n--- | ---\nleft | right\n```\n";
let mut out = Vec::new();
append_markdown_agent(src, /*width*/ None, &mut out);
let rendered = lines_to_strings(&out);
assert!(rendered.iter().any(|line| line.contains("")));
assert!(rendered.iter().any(|line| line.contains("│ A")));
assert!(!rendered.iter().any(|line| line.trim() == "A | B"));
}
#[test]
fn append_markdown_agent_unwraps_markdown_fences_for_single_column_table() {
let src = "```md\n| Only |\n|---|\n| value |\n```\n";
let mut out = Vec::new();
append_markdown_agent(src, /*width*/ None, &mut out);
let rendered = lines_to_strings(&out);
assert!(rendered.iter().any(|line| line.contains("")));
assert!(!rendered.iter().any(|line| line.trim() == "| Only |"));
}
#[test]
fn append_markdown_agent_keeps_non_markdown_fences_as_code() {
let src = "```rust\n| A | B |\n|---|---|\n| 1 | 2 |\n```\n";
let mut out = Vec::new();
append_markdown_agent(src, /*width*/ None, &mut out);
let rendered = lines_to_strings(&out);
assert_eq!(
rendered,
vec![
"| A | B |".to_string(),
"|---|---|".to_string(),
"| 1 | 2 |".to_string(),
]
);
}
#[test]
fn append_markdown_agent_unwraps_blockquoted_markdown_fence_table() {
let src = "> ```markdown\n> | A | B |\n> |---|---|\n> | 1 | 2 |\n> ```\n";
let rendered = unwrap_markdown_fences(src);
assert!(
!rendered.contains("```"),
"expected markdown fence markers to be removed: {rendered:?}"
);
}
#[test]
fn append_markdown_agent_keeps_non_blockquoted_markdown_fence_with_blockquote_table_example() {
let src = "```markdown\n> | A | B |\n> |---|---|\n> | 1 | 2 |\n```\n";
let normalized = unwrap_markdown_fences(src);
assert_eq!(normalized, src);
}
#[test]
fn append_markdown_agent_keeps_markdown_fence_when_content_is_not_table() {
let src = "```markdown\n**bold**\n```\n";
let mut out = Vec::new();
append_markdown_agent(src, /*width*/ None, &mut out);
let rendered = lines_to_strings(&out);
assert_eq!(rendered, vec!["**bold**".to_string()]);
}
#[test]
fn unwrap_markdown_fences_repro_keeps_fence_without_header_delimiter_pair() {
let src = "```markdown\n| A | B |\nnot a delimiter row\n| --- | --- |\n# Heading\n```\n";
let normalized = unwrap_markdown_fences(src);
assert_eq!(normalized, src);
}
#[test]
fn append_markdown_agent_keeps_markdown_fence_with_blank_line_between_header_and_delimiter() {
let src = "```markdown\n| A | B |\n\n|---|---|\n| 1 | 2 |\n```\n";
let rendered = unwrap_markdown_fences(src);
assert_eq!(rendered, src);
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1420,3 +1420,100 @@ fn code_block_preserves_trailing_blank_lines() {
"trailing blank line inside code fence was lost: {content:?}"
);
}
#[test]
fn table_renders_unicode_box() {
let md = "| A | B |\n|---|---|\n| 1 | 2 |\n";
let text = render_markdown_text(md);
let lines: Vec<String> = text
.lines
.iter()
.map(|line| line.spans.iter().map(|span| span.content.clone()).collect())
.collect();
assert_eq!(
lines,
vec![
"┌─────┬─────┐".to_string(),
"│ A │ B │".to_string(),
"├─────┼─────┤".to_string(),
"│ 1 │ 2 │".to_string(),
"└─────┴─────┘".to_string(),
]
);
}
#[test]
fn table_alignment_respects_markers() {
let md = "| Left | Center | Right |\n|:-----|:------:|------:|\n| a | b | c |\n";
let text = render_markdown_text(md);
let lines: Vec<String> = text
.lines
.iter()
.map(|line| line.spans.iter().map(|span| span.content.clone()).collect())
.collect();
assert_eq!(lines[1], "│ Left │ Center │ Right │");
assert_eq!(lines[3], "│ a │ b │ c │");
}
#[test]
fn table_wraps_cell_content_when_width_is_narrow() {
let md = "| Key | Description |\n| --- | --- |\n| -v | Enable very verbose logging output for debugging |\n";
let text = crate::markdown_render::render_markdown_text_with_width(md, Some(30));
let lines: Vec<String> = text
.lines
.iter()
.map(|line| line.spans.iter().map(|span| span.content.clone()).collect())
.collect();
assert!(lines[0].starts_with('┌') && lines[0].ends_with('┐'));
assert!(
lines
.iter()
.any(|line| line.contains("Enable very verbose"))
&& lines.iter().any(|line| line.contains("logging output")),
"expected wrapped row content: {lines:?}"
);
}
#[test]
fn table_inside_blockquote_has_quote_prefix() {
let md = "> | A | B |\n> |---|---|\n> | 1 | 2 |\n";
let text = render_markdown_text(md);
let lines: Vec<String> = text
.lines
.iter()
.map(|line| line.spans.iter().map(|span| span.content.clone()).collect())
.collect();
assert!(lines.iter().all(|line| line.starts_with("> ")));
assert!(lines.iter().any(|line| line.contains("┌─────┬─────┐")));
}
#[test]
fn escaped_pipes_render_in_table_cells() {
let md = "| Col |\n| --- |\n| a \\| b |\n";
let text = render_markdown_text(md);
let lines: Vec<String> = text
.lines
.iter()
.map(|line| line.spans.iter().map(|span| span.content.clone()).collect())
.collect();
assert!(lines.iter().any(|line| line.contains("a | b")));
}
#[test]
fn table_falls_back_to_pipe_rendering_if_it_cannot_fit() {
let md = "| c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 |\n|---|---|---|---|---|---|---|---|---|---|\n| 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |\n";
let text = crate::markdown_render::render_markdown_text_with_width(md, Some(20));
let lines: Vec<String> = text
.lines
.iter()
.map(|line| line.spans.iter().map(|span| span.content.clone()).collect())
.collect();
assert!(lines.first().is_some_and(|line| line.starts_with('|')));
assert!(!lines.iter().any(|line| line.contains('┌')));
}

View File

@@ -848,4 +848,41 @@ mod tests {
async fn table_like_lines_inside_fenced_code_are_not_held() {
assert_streamed_equals_full(&["```\n", "| a | b |\n", "```\n"]).await;
}
#[tokio::test]
async fn collector_source_chunks_round_trip_into_agent_fence_unwrapping() {
let deltas = [
"```md\n",
"| A | B |\n",
"|---|---|\n",
"| 1 | 2 |\n",
"```\n",
];
let mut collector =
super::MarkdownStreamCollector::new(/*width*/ None, &super::test_cwd());
let mut raw_source = String::new();
for delta in deltas {
collector.push_delta(delta);
if delta.contains('\n')
&& let Some(chunk) = collector.commit_complete_source()
{
raw_source.push_str(&chunk);
}
}
raw_source.push_str(&collector.finalize_and_drain_source());
let mut rendered = Vec::new();
crate::markdown::append_markdown_agent(&raw_source, /*width*/ None, &mut rendered);
let rendered_strs = lines_to_plain_strings(&rendered);
assert!(
rendered_strs.iter().any(|line| line.contains('┌')),
"expected markdown-fenced table to render as boxed table: {rendered_strs:?}"
);
assert!(
!rendered_strs.iter().any(|line| line.trim() == "| A | B |"),
"did not expect raw table header after markdown-fence unwrapping: {rendered_strs:?}"
);
}
}

View File

@@ -28,9 +28,12 @@ Image: alt text
———
Table below (alignment test):
| Left | Center | Right |
|:-----|:------:|------:|
| a | b | c |
┌──────┬────────┬───────┐
│ Left │ Center │ Right │
├──────┼────────┼───────┤
│ a │ b │ c │
└──────┴────────┴───────┘
Inline HTML: <sup>sup</sup> and <sub>sub</sub>.
HTML block:
<div style="border:1px solid #ccc;padding:2px">inline block</div>

File diff suppressed because it is too large Load Diff

View File

@@ -20,6 +20,7 @@ use crate::markdown_stream::MarkdownStreamCollector;
pub(crate) mod chunking;
pub(crate) mod commit_tick;
pub(crate) mod controller;
mod table_holdback;
struct QueuedLine {
line: Line<'static>,

View File

@@ -0,0 +1,210 @@
//! Pipe-table holdback scanner for source-backed agent streams.
//!
//! Agent streams with markdown tables keep the active table as mutable tail so
//! adding a row can reflow earlier table rows instead of committing a stale
//! render to scrollback.
use std::time::Instant;
use crate::table_detect::FenceKind;
use crate::table_detect::FenceTracker;
use crate::table_detect::is_table_delimiter_line;
use crate::table_detect::is_table_header_line;
use crate::table_detect::parse_table_segments;
use crate::table_detect::strip_blockquote_prefix;
/// Result of scanning accumulated raw source for pipe-table patterns.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum TableHoldbackState {
/// No table detected -- all rendered lines can flow into the stable queue.
None,
/// The last non-blank line looks like a table header row but no delimiter
/// row has followed yet. Hold back in case the next delta is a delimiter.
PendingHeader { header_start: usize },
/// A header + delimiter pair was found -- the source contains a confirmed
/// table. Content from the table header onward stays mutable.
Confirmed { table_start: usize },
}
#[derive(Clone, Copy)]
struct PreviousLineState {
source_start: usize,
fence_kind: FenceKind,
is_header: bool,
}
/// Incremental scanner for table holdback state on append-only source streams.
pub(super) struct TableHoldbackScanner {
source_offset: usize,
fence_tracker: FenceTracker,
previous_line: Option<PreviousLineState>,
pending_header_start: Option<usize>,
confirmed_table_start: Option<usize>,
}
impl TableHoldbackScanner {
pub(super) fn new() -> Self {
Self {
source_offset: 0,
fence_tracker: FenceTracker::new(),
previous_line: None,
pending_header_start: None,
confirmed_table_start: None,
}
}
pub(super) fn reset(&mut self) {
*self = Self::new();
}
pub(super) fn state(&self) -> TableHoldbackState {
if let Some(table_start) = self.confirmed_table_start {
TableHoldbackState::Confirmed { table_start }
} else if let Some(header_start) = self.pending_header_start {
TableHoldbackState::PendingHeader { header_start }
} else {
TableHoldbackState::None
}
}
pub(super) fn push_source_chunk(&mut self, source_chunk: &str) {
if source_chunk.is_empty() {
return;
}
let scan_start = Instant::now();
let mut lines = 0usize;
for source_line in source_chunk.split_inclusive('\n') {
lines += 1;
self.push_line(source_line);
}
tracing::trace!(
bytes = source_chunk.len(),
lines,
state = ?self.state(),
elapsed_us = scan_start.elapsed().as_micros(),
"table holdback incremental scan",
);
}
fn push_line(&mut self, source_line: &str) {
let line = source_line.strip_suffix('\n').unwrap_or(source_line);
let source_start = self.source_offset;
let fence_kind = self.fence_tracker.kind();
let candidate_text = if fence_kind == FenceKind::Other {
None
} else {
table_candidate_text(line)
};
let is_header = candidate_text.is_some_and(is_table_header_line);
let is_delimiter = candidate_text.is_some_and(is_table_delimiter_line);
if self.confirmed_table_start.is_none()
&& let Some(previous_line) = self.previous_line
&& previous_line.fence_kind != FenceKind::Other
&& fence_kind != FenceKind::Other
&& previous_line.is_header
&& is_delimiter
{
self.confirmed_table_start = Some(previous_line.source_start);
self.pending_header_start = None;
}
if self.confirmed_table_start.is_none() && !line.trim().is_empty() {
if fence_kind != FenceKind::Other && is_header {
self.pending_header_start = Some(source_start);
} else {
self.pending_header_start = None;
}
}
self.previous_line = Some(PreviousLineState {
source_start,
fence_kind,
is_header,
});
self.fence_tracker.advance(line);
self.source_offset = self.source_offset.saturating_add(source_line.len());
}
}
/// Strip blockquote prefixes and return the trimmed text if it contains
/// pipe-table segments, or `None` otherwise.
fn table_candidate_text(line: &str) -> Option<&str> {
let stripped = strip_blockquote_prefix(line).trim();
parse_table_segments(stripped).map(|_| stripped)
}
/// A source line annotated with whether it falls inside a fenced code block.
#[cfg(test)]
struct ParsedLine<'a> {
text: &'a str,
fence_context: FenceKind,
source_start: usize,
}
/// Parse source into lines tagged with fenced-code context for table scanning.
#[cfg(test)]
fn parse_lines_with_fence_state(source: &str) -> Vec<ParsedLine<'_>> {
let mut tracker = FenceTracker::new();
let mut lines = Vec::new();
let mut source_start = 0usize;
for raw_line in source.split('\n') {
lines.push(ParsedLine {
text: raw_line,
fence_context: tracker.kind(),
source_start,
});
tracker.advance(raw_line);
source_start = source_start
.saturating_add(raw_line.len())
.saturating_add(1);
}
lines
}
/// Scan `source` for pipe-table patterns outside of non-markdown fenced code
/// blocks.
#[cfg(test)]
pub(super) fn table_holdback_state(source: &str) -> TableHoldbackState {
let lines = parse_lines_with_fence_state(source);
for pair in lines.windows(2) {
let [header_line, delimiter_line] = pair else {
continue;
};
if header_line.fence_context == FenceKind::Other
|| delimiter_line.fence_context == FenceKind::Other
{
continue;
}
let Some(header_text) = table_candidate_text(header_line.text) else {
continue;
};
let Some(delimiter_text) = table_candidate_text(delimiter_line.text) else {
continue;
};
if is_table_header_line(header_text) && is_table_delimiter_line(delimiter_text) {
return TableHoldbackState::Confirmed {
table_start: header_line.source_start,
};
}
}
let pending_header = lines.iter().rev().find(|line| !line.text.trim().is_empty());
if let Some(line) = pending_header
&& line.fence_context != FenceKind::Other
&& table_candidate_text(line.text).is_some_and(is_table_header_line)
{
return TableHoldbackState::PendingHeader {
header_start: line.source_start,
};
}
TableHoldbackState::None
}

View File

@@ -0,0 +1,479 @@
//! Canonical pipe-table structure detection and fenced-code-block tracking for
//! raw markdown source.
//!
//! Both the streaming controller (`streaming/controller.rs`) and the
//! markdown-fence unwrapper (`markdown.rs`) need to identify pipe-table
//! structure and fenced code blocks in raw markdown source. This module
//! provides the canonical implementations so fixes only need to happen in one
//! place.
//!
//! ## Concepts
//!
//! A GFM pipe table is a sequence of lines where:
//! - A **header line** contains pipe-separated segments with at least one
//! non-empty cell.
//! - A **delimiter line** immediately follows the header and contains only
//! alignment markers (`---`, `:---`, `---:`, `:---:`), each with at least
//! three dashes.
//! - **Body rows** follow the delimiter.
//!
//! A **fenced code block** starts with 3+ backticks or tildes and ends with a
//! matching close marker. [`FenceTracker`] classifies each line as
//! [`FenceKind::Outside`], [`FenceKind::Markdown`], or [`FenceKind::Other`]
//! so callers can skip pipe characters that appear inside non-markdown fences.
//!
//! The table functions operate on single lines and do not maintain cross-line
//! state. Callers (the streaming controller and fence unwrapper) are
//! responsible for pairing consecutive lines to confirm a table.
/// Split a pipe-delimited line into trimmed segments.
///
/// Returns `None` if the line is empty or has no unescaped separator marker.
/// Leading/trailing pipes are stripped before splitting.
pub(crate) fn parse_table_segments(line: &str) -> Option<Vec<&str>> {
let trimmed = line.trim();
if trimmed.is_empty() {
return None;
}
let has_outer_pipe = trimmed.starts_with('|') || trimmed.ends_with('|');
let content = trimmed.strip_prefix('|').unwrap_or(trimmed);
let content = content.strip_suffix('|').unwrap_or(content);
let raw_segments = split_unescaped_pipe(content);
if !has_outer_pipe && raw_segments.len() <= 1 {
return None;
}
let segments: Vec<&str> = raw_segments.into_iter().map(str::trim).collect();
(!segments.is_empty()).then_some(segments)
}
/// Split `content` on unescaped `|` characters.
///
/// A pipe preceded by `\` is treated as literal text, not a column separator.
/// The backslash remains in the segment (this is structure detection, not
/// rendering).
fn split_unescaped_pipe(content: &str) -> Vec<&str> {
let mut segments = Vec::with_capacity(8);
let mut start = 0;
let bytes = content.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'\\' {
// Skip the escaped character.
i += 2;
} else if bytes[i] == b'|' {
segments.push(&content[start..i]);
start = i + 1;
i += 1;
} else {
i += 1;
}
}
segments.push(&content[start..]);
segments
}
// Small table-detection helpers inlined for the streaming hot path — they are
// called on every source line during incremental holdback scanning.
/// Whether `line` looks like a table header row (has pipe-separated
/// segments with at least one non-empty cell).
#[inline]
pub(crate) fn is_table_header_line(line: &str) -> bool {
parse_table_segments(line).is_some_and(|segments| segments.iter().any(|s| !s.is_empty()))
}
/// Whether a single segment matches the `---`, `:---`, `---:`, or `:---:`
/// alignment-colon syntax used in markdown table delimiter rows.
#[inline]
fn is_table_delimiter_segment(segment: &str) -> bool {
let trimmed = segment.trim();
if trimmed.is_empty() {
return false;
}
let without_leading = trimmed.strip_prefix(':').unwrap_or(trimmed);
let without_ends = without_leading.strip_suffix(':').unwrap_or(without_leading);
without_ends.len() >= 3 && without_ends.chars().all(|c| c == '-')
}
/// Whether `line` is a valid table delimiter row (every segment passes
/// [`is_table_delimiter_segment`]).
#[inline]
pub(crate) fn is_table_delimiter_line(line: &str) -> bool {
parse_table_segments(line)
.is_some_and(|segments| segments.into_iter().all(is_table_delimiter_segment))
}
// ---------------------------------------------------------------------------
// Fenced code block tracking
// ---------------------------------------------------------------------------
/// Where a source line sits relative to fenced code blocks.
///
/// Table holdback only applies to lines that are `Outside` or inside a
/// `Markdown` fence. Lines inside `Other` fences (e.g. `sh`, `rust`) are
/// ignored by the table scanner because their pipe characters are code, not
/// table syntax.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum FenceKind {
/// Not inside any fenced code block.
Outside,
/// Inside a `` ```md `` or `` ```markdown `` fence.
Markdown,
/// Inside a fence with a non-markdown info string.
Other,
}
/// Incremental tracker for fenced-code-block open/close transitions.
///
/// Feed lines one at a time via [`advance`](Self::advance); query the current
/// context with [`kind`](Self::kind). The tracker handles leading-whitespace
/// limits (>3 spaces → not a fence), blockquote prefix stripping, and
/// backtick/tilde marker matching.
pub(crate) struct FenceTracker {
state: Option<(char, usize, FenceKind)>,
}
impl FenceTracker {
#[inline]
pub(crate) fn new() -> Self {
Self { state: None }
}
/// Process one raw source line and update fence state.
///
/// Lines with >3 leading spaces are ignored (indented code blocks, not
/// fences). Blockquote prefixes (`>`) are stripped before scanning.
pub(crate) fn advance(&mut self, raw_line: &str) {
let leading_spaces = raw_line
.as_bytes()
.iter()
.take_while(|byte| **byte == b' ')
.count();
if leading_spaces > 3 {
return;
}
let trimmed = &raw_line[leading_spaces..];
let fence_scan_text = strip_blockquote_prefix(trimmed);
if let Some((marker, len)) = parse_fence_marker(fence_scan_text) {
if let Some((open_char, open_len, _)) = self.state {
// Close the current fence if the marker matches.
if marker == open_char
&& len >= open_len
&& fence_scan_text[len..].trim().is_empty()
{
self.state = None;
}
} else {
// Opening a new fence.
let kind = if is_markdown_fence_info(fence_scan_text, len) {
FenceKind::Markdown
} else {
FenceKind::Other
};
self.state = Some((marker, len, kind));
}
}
}
/// Current fence context for the most-recently-advanced line.
#[inline]
pub(crate) fn kind(&self) -> FenceKind {
self.state.map_or(FenceKind::Outside, |(_, _, k)| k)
}
}
/// Return fence marker character and run length for a potential fence line.
///
/// Recognises backtick and tilde fences with a minimum run of 3.
/// The input should already have leading whitespace and blockquote prefixes
/// stripped.
#[inline]
pub(crate) fn parse_fence_marker(line: &str) -> Option<(char, usize)> {
let first = line.as_bytes().first().copied()?;
if first != b'`' && first != b'~' {
return None;
}
let len = line.bytes().take_while(|&b| b == first).count();
if len < 3 {
return None;
}
Some((first as char, len))
}
/// Whether the info string after a fence marker indicates markdown content.
///
/// Matches `md` and `markdown` (case-insensitive).
#[inline]
pub(crate) fn is_markdown_fence_info(trimmed_line: &str, marker_len: usize) -> bool {
let info = trimmed_line[marker_len..]
.split_whitespace()
.next()
.unwrap_or_default();
info.eq_ignore_ascii_case("md") || info.eq_ignore_ascii_case("markdown")
}
/// Peel all leading `>` blockquote markers from a line.
///
/// Tables can appear inside blockquotes (`> | A | B |`), so the holdback
/// scanner must strip these markers before checking for table syntax.
#[inline]
pub(crate) fn strip_blockquote_prefix(line: &str) -> &str {
let mut rest = line.trim_start();
loop {
let Some(stripped) = rest.strip_prefix('>') else {
return rest;
};
rest = stripped.strip_prefix(' ').unwrap_or(stripped).trim_start();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_table_segments_basic() {
assert_eq!(
parse_table_segments("| A | B | C |"),
Some(vec!["A", "B", "C"])
);
}
#[test]
fn parse_table_segments_no_outer_pipes() {
assert_eq!(parse_table_segments("A | B | C"), Some(vec!["A", "B", "C"]));
}
#[test]
fn parse_table_segments_no_leading_pipe() {
assert_eq!(
parse_table_segments("A | B | C |"),
Some(vec!["A", "B", "C"])
);
}
#[test]
fn parse_table_segments_no_trailing_pipe() {
assert_eq!(
parse_table_segments("| A | B | C"),
Some(vec!["A", "B", "C"])
);
}
#[test]
fn parse_table_segments_single_segment_is_allowed() {
assert_eq!(parse_table_segments("| only |"), Some(vec!["only"]));
}
#[test]
fn parse_table_segments_without_pipe_returns_none() {
assert_eq!(parse_table_segments("just text"), None);
}
#[test]
fn parse_table_segments_empty_returns_none() {
assert_eq!(parse_table_segments(""), None);
assert_eq!(parse_table_segments(" "), None);
}
#[test]
fn parse_table_segments_escaped_pipe() {
// Escaped pipe should NOT split — stays inside the segment.
assert_eq!(
parse_table_segments(r"| A \| B | C |"),
Some(vec![r"A \| B", "C"])
);
}
#[test]
fn is_table_delimiter_segment_valid() {
assert!(is_table_delimiter_segment("---"));
assert!(is_table_delimiter_segment(":---"));
assert!(is_table_delimiter_segment("---:"));
assert!(is_table_delimiter_segment(":---:"));
assert!(is_table_delimiter_segment(":-------:"));
}
#[test]
fn is_table_delimiter_segment_invalid() {
assert!(!is_table_delimiter_segment(""));
assert!(!is_table_delimiter_segment("--"));
assert!(!is_table_delimiter_segment("abc"));
assert!(!is_table_delimiter_segment(":--"));
}
#[test]
fn is_table_delimiter_line_valid() {
assert!(is_table_delimiter_line("| --- | --- |"));
assert!(is_table_delimiter_line("|:---:|---:|"));
assert!(is_table_delimiter_line("--- | --- | ---"));
}
#[test]
fn is_table_delimiter_line_invalid() {
assert!(!is_table_delimiter_line("| A | B |"));
assert!(!is_table_delimiter_line("| -- | -- |"));
}
#[test]
fn is_table_header_line_valid() {
assert!(is_table_header_line("| A | B |"));
assert!(is_table_header_line("Name | Value"));
}
#[test]
fn is_table_header_line_all_empty_segments() {
assert!(!is_table_header_line("| | |"));
}
// -----------------------------------------------------------------------
// FenceTracker tests
// -----------------------------------------------------------------------
#[test]
fn fence_tracker_outside_by_default() {
let tracker = FenceTracker::new();
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_opens_and_closes_backtick_fence() {
let mut tracker = FenceTracker::new();
tracker.advance("```rust");
assert_eq!(tracker.kind(), FenceKind::Other);
tracker.advance("let x = 1;");
assert_eq!(tracker.kind(), FenceKind::Other);
tracker.advance("```");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_opens_and_closes_tilde_fence() {
let mut tracker = FenceTracker::new();
tracker.advance("~~~python");
assert_eq!(tracker.kind(), FenceKind::Other);
tracker.advance("~~~");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_markdown_fence() {
let mut tracker = FenceTracker::new();
tracker.advance("```md");
assert_eq!(tracker.kind(), FenceKind::Markdown);
tracker.advance("| A | B |");
assert_eq!(tracker.kind(), FenceKind::Markdown);
tracker.advance("```");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_markdown_case_insensitive() {
let mut tracker = FenceTracker::new();
tracker.advance("```Markdown");
assert_eq!(tracker.kind(), FenceKind::Markdown);
tracker.advance("```");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_nested_shorter_marker_does_not_close() {
let mut tracker = FenceTracker::new();
tracker.advance("````sh");
assert_eq!(tracker.kind(), FenceKind::Other);
// Shorter marker inside should not close.
tracker.advance("```");
assert_eq!(tracker.kind(), FenceKind::Other);
// Matching length closes.
tracker.advance("````");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_mismatched_char_does_not_close() {
let mut tracker = FenceTracker::new();
tracker.advance("```sh");
assert_eq!(tracker.kind(), FenceKind::Other);
// Tilde marker should not close a backtick fence.
tracker.advance("~~~");
assert_eq!(tracker.kind(), FenceKind::Other);
tracker.advance("```");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_indented_4_spaces_ignored() {
let mut tracker = FenceTracker::new();
tracker.advance(" ```sh");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_blockquote_prefix_stripped() {
let mut tracker = FenceTracker::new();
tracker.advance("> ```sh");
assert_eq!(tracker.kind(), FenceKind::Other);
tracker.advance("> ```");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
#[test]
fn fence_tracker_close_with_trailing_content_does_not_close() {
let mut tracker = FenceTracker::new();
tracker.advance("```sh");
assert_eq!(tracker.kind(), FenceKind::Other);
// Trailing content prevents closing.
tracker.advance("``` extra");
assert_eq!(tracker.kind(), FenceKind::Other);
tracker.advance("```");
assert_eq!(tracker.kind(), FenceKind::Outside);
}
// -----------------------------------------------------------------------
// Fence helper function tests
// -----------------------------------------------------------------------
#[test]
fn parse_fence_marker_backtick() {
assert_eq!(parse_fence_marker("```rust"), Some(('`', 3)));
assert_eq!(parse_fence_marker("````"), Some(('`', 4)));
}
#[test]
fn parse_fence_marker_tilde() {
assert_eq!(parse_fence_marker("~~~python"), Some(('~', 3)));
}
#[test]
fn parse_fence_marker_too_short() {
assert_eq!(parse_fence_marker("``"), None);
assert_eq!(parse_fence_marker("~~"), None);
}
#[test]
fn parse_fence_marker_not_fence() {
assert_eq!(parse_fence_marker("hello"), None);
assert_eq!(parse_fence_marker(""), None);
}
#[test]
fn is_markdown_fence_info_basic() {
assert!(is_markdown_fence_info("```md", /*marker_len*/ 3));
assert!(is_markdown_fence_info("```markdown", /*marker_len*/ 3));
assert!(is_markdown_fence_info("```MD", /*marker_len*/ 3));
assert!(!is_markdown_fence_info("```rust", /*marker_len*/ 3));
assert!(!is_markdown_fence_info("```", /*marker_len*/ 3));
}
#[test]
fn strip_blockquote_prefix_basic() {
assert_eq!(strip_blockquote_prefix("> hello"), "hello");
assert_eq!(strip_blockquote_prefix("> > nested"), "nested");
assert_eq!(strip_blockquote_prefix("no prefix"), "no prefix");
}
}