Compare commits

...

3 Commits

Author SHA1 Message Date
Shoubhit Dash
96b58aadd6 Merge branch 'dev' into feat/fff-search-tools 2026-03-20 22:27:25 +05:30
Shoubhit Dash
2948b2fb73 feat(tools): switch search tools to fff 2026-03-20 22:25:20 +05:30
Shoubhit Dash
b5a7ad7085 feat(file): add fff-backed search 2026-03-20 22:25:05 +05:30
20 changed files with 980 additions and 644 deletions

View File

@@ -325,6 +325,7 @@
"@aws-sdk/credential-providers": "3.993.0",
"@clack/prompts": "1.0.0-alpha.1",
"@effect/platform-node": "catalog:",
"@ff-labs/fff-node": "0.4.2",
"@gitlab/gitlab-ai-provider": "3.6.0",
"@gitlab/opencode-gitlab-auth": "1.3.3",
"@hono/standard-validator": "0.1.5",
@@ -345,7 +346,6 @@
"@solid-primitives/event-bus": "1.1.2",
"@solid-primitives/scheduled": "1.5.2",
"@standard-schema/spec": "1.0.0",
"@zip.js/zip.js": "2.7.62",
"ai": "catalog:",
"ai-gateway-provider": "2.3.1",
"bonjour-service": "1.3.0",
@@ -1098,6 +1098,24 @@
"@fastify/rate-limit": ["@fastify/rate-limit@10.3.0", "", { "dependencies": { "@lukeed/ms": "^2.0.2", "fastify-plugin": "^5.0.0", "toad-cache": "^3.7.0" } }, "sha512-eIGkG9XKQs0nyynatApA3EVrojHOuq4l6fhB4eeCk4PIOeadvOJz9/4w3vGI44Go17uaXOWEcPkaD8kuKm7g6Q=="],
"@ff-labs/fff-bin-darwin-arm64": ["@ff-labs/fff-bin-darwin-arm64@0.4.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-R9ieZvBaAmYNbdGT3gs2HUQ0Sm4I5tBrJwOepdCoeIZvJFI71hCY2DCFzeoXH2wbxMsPF70c1FSr8qERhcrbVw=="],
"@ff-labs/fff-bin-darwin-x64": ["@ff-labs/fff-bin-darwin-x64@0.4.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-iJNgUdkS1sVMhWe6l60ZmG9BcSB87CdO65K4AuMbwHQZTHxje9Sapf+AWPGYem6H0endS7HF7ejH+yoZmCF0uw=="],
"@ff-labs/fff-bin-linux-arm64-gnu": ["@ff-labs/fff-bin-linux-arm64-gnu@0.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-vldJC/j/Kf0LVS599CPTeIaBAd+8J6FFJ1euWn4OoSu63P3CD+9ITrmPWkIGUrt+0myOXABAx0KgLBGADtIAKg=="],
"@ff-labs/fff-bin-linux-arm64-musl": ["@ff-labs/fff-bin-linux-arm64-musl@0.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-RgL1Oq6QMZm+M4R14SYLtiRMUObA8k+EHIftaplKpLu4Cr0q5lCclRszS0o0Le0hmFrrIvMn6pFRE7LoEzKqAQ=="],
"@ff-labs/fff-bin-linux-x64-gnu": ["@ff-labs/fff-bin-linux-x64-gnu@0.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-ysImURWrxLT7WFTn46NrXOg4ygbuIp4NkKbWzOAzLYoMOU5JRllUxb3huw3sZNbXn+/9tpq3OE9VmWuAi0YZ/w=="],
"@ff-labs/fff-bin-linux-x64-musl": ["@ff-labs/fff-bin-linux-x64-musl@0.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-Us4ysq/oCrcf+h5lOPzDbxFJ8WI8bSVbSVudYHYFpe54417oWtbokdbzgb5Yx7108dW7jCDtGkxq+Cnau2002A=="],
"@ff-labs/fff-bin-win32-arm64": ["@ff-labs/fff-bin-win32-arm64@0.4.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-rkF8lNEUhaZmGESJdQGSnIReg5hMDbb7giFxTlEzgeFRkWZpjKkDajGVuJ+Rd2bI5AoxUNuTDUfvta5EkQ2S5g=="],
"@ff-labs/fff-bin-win32-x64": ["@ff-labs/fff-bin-win32-x64@0.4.2", "", { "os": "win32", "cpu": "x64" }, "sha512-wtSZiI2/7Z61GdVlGxPtXcuQV4EyoHgVBLhJ5wXcGwEQLp/r8GUWzSpN7iDQaOKKEvHbT2XiEbcbdw+jhDR7qQ=="],
"@ff-labs/fff-node": ["@ff-labs/fff-node@0.4.2", "", { "dependencies": { "ffi-rs": "^1.0.0" }, "optionalDependencies": { "@ff-labs/fff-bin-darwin-arm64": "0.4.2", "@ff-labs/fff-bin-darwin-x64": "0.4.2", "@ff-labs/fff-bin-linux-arm64-gnu": "0.4.2", "@ff-labs/fff-bin-linux-arm64-musl": "0.4.2", "@ff-labs/fff-bin-linux-x64-gnu": "0.4.2", "@ff-labs/fff-bin-linux-x64-musl": "0.4.2", "@ff-labs/fff-bin-win32-arm64": "0.4.2", "@ff-labs/fff-bin-win32-x64": "0.4.2" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ] }, "sha512-R1jjyvWmLC6qLOxFwdZhhA4UrOZY6r5nuqsuMpdsrDOhMMktJsbhMDZzRqXIy+GXTQBqAF1oBhW6FN6ahTCPBA=="],
"@floating-ui/core": ["@floating-ui/core@1.7.5", "", { "dependencies": { "@floating-ui/utils": "^0.2.11" } }, "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ=="],
"@floating-ui/dom": ["@floating-ui/dom@1.7.6", "", { "dependencies": { "@floating-ui/core": "^1.7.5", "@floating-ui/utils": "^0.2.11" } }, "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ=="],
@@ -2234,7 +2252,27 @@
"@xmldom/xmldom": ["@xmldom/xmldom@0.8.11", "", {}, "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw=="],
"@zip.js/zip.js": ["@zip.js/zip.js@2.7.62", "", {}, "sha512-OaLvZ8j4gCkLn048ypkZu29KX30r8/OfFF2w4Jo5WXFr+J04J+lzJ5TKZBVgFXhlvSkqNFQdfnY1Q8TMTCyBVA=="],
"@yuuang/ffi-rs-android-arm64": ["@yuuang/ffi-rs-android-arm64@1.3.1", "", { "os": "android", "cpu": "arm64" }, "sha512-V4nmlXdOYZEa7GOxSExVG95SLp8FE0iTq2yKeN54UlfNMr3Sik+1Ff57LcCv7qYcn4TBqnBAt5rT3FAM6T6caQ=="],
"@yuuang/ffi-rs-darwin-arm64": ["@yuuang/ffi-rs-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YlnTMIyzfW3mAULC5ZA774nzQfFlYXM0rrfq/8ZzWt+IMbYk55a++jrI+6JeKV+1EqlDS3TFBEFtjdBNG94KzQ=="],
"@yuuang/ffi-rs-darwin-x64": ["@yuuang/ffi-rs-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-sI3LpQQ34SX4nyOHc5yxA7FSqs9qPEUMqW/y/wWo9cuyPpaHMFsi/BeOVYsnC0syp3FrY7gzn6RnD6PlXCktXg=="],
"@yuuang/ffi-rs-linux-arm-gnueabihf": ["@yuuang/ffi-rs-linux-arm-gnueabihf@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-1WkcGkJTlwh4ZA59htKI+RXhiL3oKiYwLv7PO8LUf6FuADK73s5GcXp67iakKu243uYu+qGYr4RHco4ySddYhQ=="],
"@yuuang/ffi-rs-linux-arm64-gnu": ["@yuuang/ffi-rs-linux-arm64-gnu@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-J2PwqviycZxaEVA0Bwv38LqGDGSB9A1DPN4iYginYJZSvTvKW8kh7Tis0HbZrX1YDKnY8hi3lt0N0tCTNPDH5Q=="],
"@yuuang/ffi-rs-linux-arm64-musl": ["@yuuang/ffi-rs-linux-arm64-musl@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-Hn1W1hBPssTaqikU1Bqp1XUdDdOgbnYVIOtR++LVx66hhrtjf/xrIUQOhTm+NmOFDG16JUKXe1skfM4gpaqYwg=="],
"@yuuang/ffi-rs-linux-x64-gnu": ["@yuuang/ffi-rs-linux-x64-gnu@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-kW6e+oCYZPvpH2ppPsffA18e1aLowtmWTRjVlyHtY04g/nQDepQvDUkkcvInh9fW5jLna7PjHvktW1tVgYIj2A=="],
"@yuuang/ffi-rs-linux-x64-musl": ["@yuuang/ffi-rs-linux-x64-musl@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-HTwblAzruUS16nQPrez3ozvEHm1Xxh8J8w7rZYrpmAcNl1hzyOT8z/hY70M9Rt9fOqQ4Ovgor9qVy/U3ZJo0ZA=="],
"@yuuang/ffi-rs-win32-arm64-msvc": ["@yuuang/ffi-rs-win32-arm64-msvc@1.3.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-WeZkGl2BP1U4tRhEQH+FXLQS52N8obp74smK5AAGOfzPAT1pHkq6+dVkC1QCSIt7dHJs7SPtlnQw+5DkdZYlWA=="],
"@yuuang/ffi-rs-win32-ia32-msvc": ["@yuuang/ffi-rs-win32-ia32-msvc@1.3.1", "", { "os": "win32", "cpu": [ "x64", "ia32", ] }, "sha512-rNGgMeCH5mdeHiMiJgt7wWXovZ+FHEfXhU9p4zZBH4n8M1/QnEsRUwlapISPLpILSGpoYS6iBuq9/fUlZY8Mhg=="],
"@yuuang/ffi-rs-win32-x64-msvc": ["@yuuang/ffi-rs-win32-x64-msvc@1.3.1", "", { "os": "win32", "cpu": "x64" }, "sha512-dr2LcLD2CXo2a7BktlOpV68QhayqiI112KxIJC9tBgQO/Dkdg4CPsdqmvzzLhFo64iC5RLl2BT7M5lJImrfUWw=="],
"abbrev": ["abbrev@2.0.0", "", {}, "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ=="],
@@ -2934,6 +2972,8 @@
"fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="],
"ffi-rs": ["ffi-rs@1.3.1", "", { "optionalDependencies": { "@yuuang/ffi-rs-android-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-x64": "1.3.1", "@yuuang/ffi-rs-linux-arm-gnueabihf": "1.3.1", "@yuuang/ffi-rs-linux-arm64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-arm64-musl": "1.3.1", "@yuuang/ffi-rs-linux-x64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-x64-musl": "1.3.1", "@yuuang/ffi-rs-win32-arm64-msvc": "1.3.1", "@yuuang/ffi-rs-win32-ia32-msvc": "1.3.1", "@yuuang/ffi-rs-win32-x64-msvc": "1.3.1" } }, "sha512-ZyNXL9fnclnZV+waQmWB9JrfbIEyxQa1OWtMrHOrAgcC04PgP5hBMG5TdhVN8N4uT/eul8zCFMVnJUukAFFlXA=="],
"file-type": ["file-type@16.5.4", "", { "dependencies": { "readable-web-to-node-stream": "^3.0.0", "strtok3": "^6.2.4", "token-types": "^4.1.1" } }, "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw=="],
"filelist": ["filelist@1.0.6", "", { "dependencies": { "minimatch": "^5.0.1" } }, "sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA=="],

View File

@@ -90,6 +90,7 @@
"@aws-sdk/credential-providers": "3.993.0",
"@clack/prompts": "1.0.0-alpha.1",
"@effect/platform-node": "catalog:",
"@ff-labs/fff-node": "0.4.2",
"@gitlab/gitlab-ai-provider": "3.6.0",
"@gitlab/opencode-gitlab-auth": "1.3.3",
"@hono/standard-validator": "0.1.5",
@@ -110,7 +111,6 @@
"@solid-primitives/event-bus": "1.1.2",
"@solid-primitives/scheduled": "1.5.2",
"@standard-schema/spec": "1.0.0",
"@zip.js/zip.js": "2.7.62",
"ai": "catalog:",
"ai-gateway-provider": "2.3.1",
"bonjour-service": "1.3.0",

View File

@@ -2,7 +2,7 @@ import { EOL } from "os"
import { File } from "../../../file"
import { bootstrap } from "../../bootstrap"
import { cmd } from "../cmd"
import { Ripgrep } from "@/file/ripgrep"
import { Fff } from "@/file/fff"
const FileSearchCommand = cmd({
command: "search <query>",
@@ -77,7 +77,7 @@ const FileTreeCommand = cmd({
default: process.cwd(),
}),
async handler(args) {
const files = await Ripgrep.tree({ cwd: args.dir, limit: 200 })
const files = await Fff.tree({ cwd: args.dir, limit: 200 })
console.log(JSON.stringify(files, null, 2))
},
})

View File

@@ -4,7 +4,7 @@ import { cmd } from "../cmd"
import { ConfigCommand } from "./config"
import { FileCommand } from "./file"
import { LSPCommand } from "./lsp"
import { RipgrepCommand } from "./ripgrep"
import { SearchCommand } from "./search"
import { ScrapCommand } from "./scrap"
import { SkillCommand } from "./skill"
import { SnapshotCommand } from "./snapshot"
@@ -17,7 +17,7 @@ export const DebugCommand = cmd({
yargs
.command(ConfigCommand)
.command(LSPCommand)
.command(RipgrepCommand)
.command(SearchCommand)
.command(FileCommand)
.command(ScrapCommand)
.command(SkillCommand)

View File

@@ -1,33 +1,34 @@
import { EOL } from "os"
import { Ripgrep } from "../../../file/ripgrep"
import { Fff } from "../../../file/fff"
import { Instance } from "../../../project/instance"
import { bootstrap } from "../../bootstrap"
import { cmd } from "../cmd"
import { Glob } from "@/util/glob"
export const RipgrepCommand = cmd({
command: "rg",
describe: "ripgrep debugging utilities",
builder: (yargs) => yargs.command(TreeCommand).command(FilesCommand).command(SearchCommand).demandCommand(),
export const SearchCommand = cmd({
command: "search",
describe: "fff search debugging utilities",
builder: (yargs) => yargs.command(TreeCommand).command(FilesCommand).command(ContentCommand).demandCommand(),
async handler() {},
})
const TreeCommand = cmd({
command: "tree",
describe: "show file tree using ripgrep",
describe: "show file tree using fff",
builder: (yargs) =>
yargs.option("limit", {
type: "number",
}),
async handler(args) {
await bootstrap(process.cwd(), async () => {
process.stdout.write((await Ripgrep.tree({ cwd: Instance.directory, limit: args.limit })) + EOL)
process.stdout.write((await Fff.tree({ cwd: Instance.directory, limit: args.limit })) + EOL)
})
},
})
const FilesCommand = cmd({
command: "files",
describe: "list files using ripgrep",
describe: "list files using fff",
builder: (yargs) =>
yargs
.option("query", {
@@ -44,22 +45,24 @@ const FilesCommand = cmd({
}),
async handler(args) {
await bootstrap(process.cwd(), async () => {
const files: string[] = []
for await (const file of Ripgrep.files({
const limit = args.limit ?? 100
const files = (await Glob.scan("**/*", {
cwd: Instance.directory,
glob: args.glob ? [args.glob] : undefined,
})) {
files.push(file)
if (args.limit && files.length >= args.limit) break
}
include: "file",
dot: true,
}))
.map((x) => x.replaceAll("\\", "/"))
.filter((x) => Fff.allowed({ rel: x, hidden: true, glob: args.glob ? [args.glob] : undefined }))
.filter((x) => !args.query || x.includes(args.query))
.slice(0, limit)
process.stdout.write(files.join(EOL) + EOL)
})
},
})
const SearchCommand = cmd({
command: "search <pattern>",
describe: "search file contents using ripgrep",
const ContentCommand = cmd({
command: "content <pattern>",
describe: "search file contents using fff",
builder: (yargs) =>
yargs
.positional("pattern", {
@@ -76,12 +79,12 @@ const SearchCommand = cmd({
description: "Limit number of results",
}),
async handler(args) {
const results = await Ripgrep.search({
const rows = await Fff.search({
cwd: process.cwd(),
pattern: args.pattern,
glob: args.glob as string[] | undefined,
limit: args.limit,
})
process.stdout.write(JSON.stringify(results, null, 2) + EOL)
process.stdout.write(JSON.stringify(rows, null, 2) + EOL)
},
})

View File

@@ -0,0 +1,273 @@
import fs from "fs/promises"
import path from "path"
import {
FileFinder,
type FileItem,
type GrepCursor,
type GrepMatch,
type GrepMode,
type SearchResult,
} from "@ff-labs/fff-node"
import z from "zod"
import { Global } from "../global"
import { Instance } from "../project/instance"
import { Filesystem } from "../util/filesystem"
import { Glob } from "../util/glob"
import { Log } from "../util/log"
export namespace Fff {
const log = Log.create({ service: "file.fff" })
export const Match = z.object({
path: z.object({
text: z.string(),
}),
lines: z.object({
text: z.string(),
}),
line_number: z.number(),
absolute_offset: z.number(),
submatches: z.array(
z.object({
match: z.object({
text: z.string(),
}),
start: z.number(),
end: z.number(),
}),
),
})
const state = Instance.state(
async () => ({
map: new Map<string, FileFinder>(),
pending: new Map<string, Promise<FileFinder>>(),
}),
async (state) => {
for (const pick of state.map.values()) pick.destroy()
},
)
const root = path.join(Global.Path.cache, "fff")
function key(dir: string) {
return Buffer.from(dir).toString("base64url")
}
async function db(dir: string) {
await fs.mkdir(root, { recursive: true })
const id = key(dir)
return {
frecency: path.join(root, `${id}.frecency.mdb`),
history: path.join(root, `${id}.history.mdb`),
}
}
function refresh(pick: FileFinder) {
const git = pick.refreshGitStatus()
if (!git.ok) {
log.warn("git refresh failed", { error: git.error })
return
}
}
export async function picker(cwd: string) {
const dir = Filesystem.resolve(cwd)
const memo = await state()
const cached = memo.map.get(dir)
if (cached) return cached
const wait = memo.pending.get(dir)
if (wait) return wait
const next = (async () => {
const files = await db(dir)
const made = FileFinder.create({
basePath: dir,
frecencyDbPath: files.frecency,
historyDbPath: files.history,
aiMode: true,
})
if (!made.ok) throw new Error(made.error)
const pick = made.value
const done = await pick.waitForScan(5000)
if (!done.ok) {
pick.destroy()
throw new Error(done.error)
}
memo.map.set(dir, pick)
refresh(pick)
return pick
})()
memo.pending.set(dir, next)
try {
return await next
} finally {
if (memo.pending.get(dir) === next) memo.pending.delete(dir)
}
}
export async function files(input: { cwd: string; query: string; page?: number; size?: number; current?: string }) {
const pick = await picker(input.cwd)
const out = pick.fileSearch(input.query, {
pageIndex: input.page ?? 0,
pageSize: input.size ?? 100,
currentFile: input.current,
})
if (!out.ok) throw new Error(out.error)
return out.value
}
export async function grep(input: {
cwd: string
query: string
mode?: GrepMode
max?: number
before?: number
after?: number
budget?: number
cursor?: GrepCursor | null
}) {
const pick = await picker(input.cwd)
const out = pick.grep(input.query, {
mode: input.mode,
maxMatchesPerFile: input.max,
beforeContext: input.before,
afterContext: input.after,
timeBudgetMs: input.budget,
cursor: input.cursor,
})
if (!out.ok) throw new Error(out.error)
return out.value
}
function norm(text: string) {
return text.replaceAll("\\", "/")
}
function hidden(rel: string) {
return norm(rel)
.split("/")
.some((part) => part.startsWith("."))
}
function accept(rel: string, file: string, glob?: string[], show?: boolean) {
if (show === false && hidden(rel)) return false
if (!glob?.length) return true
const allow = glob.filter((x) => !x.startsWith("!"))
const deny = glob.filter((x) => x.startsWith("!")).map((x) => x.slice(1))
if (allow.length > 0 && !allow.some((x) => Glob.match(x, rel) || Glob.match(x, file))) return false
if (deny.some((x) => Glob.match(x, rel) || Glob.match(x, file))) return false
return true
}
export function allowed(input: { rel: string; file?: string; glob?: string[]; hidden?: boolean }) {
return accept(input.rel, input.file ?? input.rel.split("/").at(-1) ?? input.rel, input.glob, input.hidden !== false)
}
export async function tree(input: { cwd: string; limit?: number; signal?: AbortSignal }) {
input.signal?.throwIfAborted()
const files = (await Glob.scan("**/*", {
cwd: input.cwd,
include: "file",
dot: true,
}))
.map((row) => norm(row))
.filter((row) => allowed({ rel: row, hidden: true }))
.toSorted((a, b) => a.localeCompare(b))
input.signal?.throwIfAborted()
interface Node {
name: string
children: Map<string, Node>
}
function dir(node: Node, name: string) {
const old = node.children.get(name)
if (old) return old
const next = { name, children: new Map<string, Node>() }
node.children.set(name, next)
return next
}
const root = { name: "", children: new Map<string, Node>() }
for (const file of files) {
if (file.includes(".opencode")) continue
const parts = file.split("/")
if (parts.length < 2) continue
let node = root
for (const part of parts.slice(0, -1)) {
node = dir(node, part)
}
}
function count(node: Node): number {
return Array.from(node.children.values()).reduce((sum, child) => sum + 1 + count(child), 0)
}
const total = count(root)
const limit = input.limit ?? total
const lines: string[] = []
const queue = Array.from(root.children.values())
.toSorted((a, b) => a.name.localeCompare(b.name))
.map((node) => ({ node, path: node.name }))
let used = 0
for (let i = 0; i < queue.length && used < limit; i++) {
input.signal?.throwIfAborted()
const row = queue[i]
lines.push(row.path)
used++
queue.push(
...Array.from(row.node.children.values())
.toSorted((a, b) => a.name.localeCompare(b.name))
.map((node) => ({ node, path: `${row.path}/${node.name}` })),
)
}
if (total > used) lines.push(`[${total - used} truncated]`)
input.signal?.throwIfAborted()
return lines.join("\n")
}
export async function search(input: {
cwd: string
pattern: string
glob?: string[]
limit?: number
follow?: boolean
}) {
const out = await grep({
cwd: input.cwd,
query: input.pattern,
mode: "regex",
max: input.limit,
})
const rows = out.items
.filter((row) => accept(norm(row.relativePath), row.fileName, input.glob, true))
.slice(0, input.limit)
.map((row) => ({
path: { text: row.relativePath },
lines: { text: row.lineContent },
line_number: row.lineNumber,
absolute_offset: row.byteOffset,
submatches: row.matchRanges
.map(([start, end]) => {
const text = row.lineContent.slice(start, end)
if (!text) return undefined
return {
match: { text },
start,
end,
}
})
.filter((row) => row !== undefined),
}))
return Match.array().parse(rows)
}
export type Search = SearchResult
export type File = FileItem
export type Hit = GrepMatch
}

View File

@@ -12,9 +12,9 @@ import z from "zod"
import { Global } from "../global"
import { Instance } from "../project/instance"
import { Filesystem } from "../util/filesystem"
import { Glob } from "../util/glob"
import { Log } from "../util/log"
import { Protected } from "./protected"
import { Ripgrep } from "./ripgrep"
export namespace File {
export const Info = z
@@ -401,7 +401,11 @@ export namespace File {
next.dirs = Array.from(dirs).toSorted()
} else {
const seen = new Set<string>()
for await (const file of Ripgrep.files({ cwd: instance.directory })) {
for (const file of (await Glob.scan("**/*", {
cwd: instance.directory,
include: "file",
dot: true,
})).toSorted((a, b) => a.localeCompare(b))) {
next.files.push(file)
let current = file
while (true) {

View File

@@ -1,376 +0,0 @@
// Ripgrep utility functions
import path from "path"
import { Global } from "../global"
import fs from "fs/promises"
import z from "zod"
import { NamedError } from "@opencode-ai/util/error"
import { lazy } from "../util/lazy"
import { Filesystem } from "../util/filesystem"
import { Process } from "../util/process"
import { which } from "../util/which"
import { text } from "node:stream/consumers"
import { ZipReader, BlobReader, BlobWriter } from "@zip.js/zip.js"
import { Log } from "@/util/log"
export namespace Ripgrep {
const log = Log.create({ service: "ripgrep" })
const Stats = z.object({
elapsed: z.object({
secs: z.number(),
nanos: z.number(),
human: z.string(),
}),
searches: z.number(),
searches_with_match: z.number(),
bytes_searched: z.number(),
bytes_printed: z.number(),
matched_lines: z.number(),
matches: z.number(),
})
const Begin = z.object({
type: z.literal("begin"),
data: z.object({
path: z.object({
text: z.string(),
}),
}),
})
export const Match = z.object({
type: z.literal("match"),
data: z.object({
path: z.object({
text: z.string(),
}),
lines: z.object({
text: z.string(),
}),
line_number: z.number(),
absolute_offset: z.number(),
submatches: z.array(
z.object({
match: z.object({
text: z.string(),
}),
start: z.number(),
end: z.number(),
}),
),
}),
})
const End = z.object({
type: z.literal("end"),
data: z.object({
path: z.object({
text: z.string(),
}),
binary_offset: z.number().nullable(),
stats: Stats,
}),
})
const Summary = z.object({
type: z.literal("summary"),
data: z.object({
elapsed_total: z.object({
human: z.string(),
nanos: z.number(),
secs: z.number(),
}),
stats: Stats,
}),
})
const Result = z.union([Begin, Match, End, Summary])
export type Result = z.infer<typeof Result>
export type Match = z.infer<typeof Match>
export type Begin = z.infer<typeof Begin>
export type End = z.infer<typeof End>
export type Summary = z.infer<typeof Summary>
const PLATFORM = {
"arm64-darwin": { platform: "aarch64-apple-darwin", extension: "tar.gz" },
"arm64-linux": {
platform: "aarch64-unknown-linux-gnu",
extension: "tar.gz",
},
"x64-darwin": { platform: "x86_64-apple-darwin", extension: "tar.gz" },
"x64-linux": { platform: "x86_64-unknown-linux-musl", extension: "tar.gz" },
"arm64-win32": { platform: "aarch64-pc-windows-msvc", extension: "zip" },
"x64-win32": { platform: "x86_64-pc-windows-msvc", extension: "zip" },
} as const
export const ExtractionFailedError = NamedError.create(
"RipgrepExtractionFailedError",
z.object({
filepath: z.string(),
stderr: z.string(),
}),
)
export const UnsupportedPlatformError = NamedError.create(
"RipgrepUnsupportedPlatformError",
z.object({
platform: z.string(),
}),
)
export const DownloadFailedError = NamedError.create(
"RipgrepDownloadFailedError",
z.object({
url: z.string(),
status: z.number(),
}),
)
const state = lazy(async () => {
const system = which("rg")
if (system) {
const stat = await fs.stat(system).catch(() => undefined)
if (stat?.isFile()) return { filepath: system }
log.warn("bun.which returned invalid rg path", { filepath: system })
}
const filepath = path.join(Global.Path.bin, "rg" + (process.platform === "win32" ? ".exe" : ""))
if (!(await Filesystem.exists(filepath))) {
const platformKey = `${process.arch}-${process.platform}` as keyof typeof PLATFORM
const config = PLATFORM[platformKey]
if (!config) throw new UnsupportedPlatformError({ platform: platformKey })
const version = "14.1.1"
const filename = `ripgrep-${version}-${config.platform}.${config.extension}`
const url = `https://github.com/BurntSushi/ripgrep/releases/download/${version}/${filename}`
const response = await fetch(url)
if (!response.ok) throw new DownloadFailedError({ url, status: response.status })
const arrayBuffer = await response.arrayBuffer()
const archivePath = path.join(Global.Path.bin, filename)
await Filesystem.write(archivePath, Buffer.from(arrayBuffer))
if (config.extension === "tar.gz") {
const args = ["tar", "-xzf", archivePath, "--strip-components=1"]
if (platformKey.endsWith("-darwin")) args.push("--include=*/rg")
if (platformKey.endsWith("-linux")) args.push("--wildcards", "*/rg")
const proc = Process.spawn(args, {
cwd: Global.Path.bin,
stderr: "pipe",
stdout: "pipe",
})
const exit = await proc.exited
if (exit !== 0) {
const stderr = proc.stderr ? await text(proc.stderr) : ""
throw new ExtractionFailedError({
filepath,
stderr,
})
}
}
if (config.extension === "zip") {
const zipFileReader = new ZipReader(new BlobReader(new Blob([arrayBuffer])))
const entries = await zipFileReader.getEntries()
let rgEntry: any
for (const entry of entries) {
if (entry.filename.endsWith("rg.exe")) {
rgEntry = entry
break
}
}
if (!rgEntry) {
throw new ExtractionFailedError({
filepath: archivePath,
stderr: "rg.exe not found in zip archive",
})
}
const rgBlob = await rgEntry.getData(new BlobWriter())
if (!rgBlob) {
throw new ExtractionFailedError({
filepath: archivePath,
stderr: "Failed to extract rg.exe from zip archive",
})
}
await Filesystem.write(filepath, Buffer.from(await rgBlob.arrayBuffer()))
await zipFileReader.close()
}
await fs.unlink(archivePath)
if (!platformKey.endsWith("-win32")) await fs.chmod(filepath, 0o755)
}
return {
filepath,
}
})
export async function filepath() {
const { filepath } = await state()
return filepath
}
export async function* files(input: {
cwd: string
glob?: string[]
hidden?: boolean
follow?: boolean
maxDepth?: number
signal?: AbortSignal
}) {
input.signal?.throwIfAborted()
const args = [await filepath(), "--files", "--glob=!.git/*"]
if (input.follow) args.push("--follow")
if (input.hidden !== false) args.push("--hidden")
if (input.maxDepth !== undefined) args.push(`--max-depth=${input.maxDepth}`)
if (input.glob) {
for (const g of input.glob) {
args.push(`--glob=${g}`)
}
}
// Guard against invalid cwd to provide a consistent ENOENT error.
if (!(await fs.stat(input.cwd).catch(() => undefined))?.isDirectory()) {
throw Object.assign(new Error(`No such file or directory: '${input.cwd}'`), {
code: "ENOENT",
errno: -2,
path: input.cwd,
})
}
const proc = Process.spawn(args, {
cwd: input.cwd,
stdout: "pipe",
stderr: "ignore",
abort: input.signal,
})
if (!proc.stdout) {
throw new Error("Process output not available")
}
let buffer = ""
const stream = proc.stdout as AsyncIterable<Buffer | string>
for await (const chunk of stream) {
input.signal?.throwIfAborted()
buffer += typeof chunk === "string" ? chunk : chunk.toString()
// Handle both Unix (\n) and Windows (\r\n) line endings
const lines = buffer.split(/\r?\n/)
buffer = lines.pop() || ""
for (const line of lines) {
if (line) yield line
}
}
if (buffer) yield buffer
await proc.exited
input.signal?.throwIfAborted()
}
export async function tree(input: { cwd: string; limit?: number; signal?: AbortSignal }) {
log.info("tree", input)
const files = await Array.fromAsync(Ripgrep.files({ cwd: input.cwd, signal: input.signal }))
interface Node {
name: string
children: Map<string, Node>
}
function dir(node: Node, name: string) {
const existing = node.children.get(name)
if (existing) return existing
const next = { name, children: new Map() }
node.children.set(name, next)
return next
}
const root: Node = { name: "", children: new Map() }
for (const file of files) {
if (file.includes(".opencode")) continue
const parts = file.split(path.sep)
if (parts.length < 2) continue
let node = root
for (const part of parts.slice(0, -1)) {
node = dir(node, part)
}
}
function count(node: Node): number {
let total = 0
for (const child of node.children.values()) {
total += 1 + count(child)
}
return total
}
const total = count(root)
const limit = input.limit ?? total
const lines: string[] = []
const queue: { node: Node; path: string }[] = []
for (const child of Array.from(root.children.values()).sort((a, b) => a.name.localeCompare(b.name))) {
queue.push({ node: child, path: child.name })
}
let used = 0
for (let i = 0; i < queue.length && used < limit; i++) {
const { node, path } = queue[i]
lines.push(path)
used++
for (const child of Array.from(node.children.values()).sort((a, b) => a.name.localeCompare(b.name))) {
queue.push({ node: child, path: `${path}/${child.name}` })
}
}
if (total > used) lines.push(`[${total - used} truncated]`)
return lines.join("\n")
}
export async function search(input: {
cwd: string
pattern: string
glob?: string[]
limit?: number
follow?: boolean
}) {
const args = [`${await filepath()}`, "--json", "--hidden", "--glob=!.git/*"]
if (input.follow) args.push("--follow")
if (input.glob) {
for (const g of input.glob) {
args.push(`--glob=${g}`)
}
}
if (input.limit) {
args.push(`--max-count=${input.limit}`)
}
args.push("--")
args.push(input.pattern)
const result = await Process.text(args, {
cwd: input.cwd,
nothrow: true,
})
if (result.code !== 0) {
return []
}
// Handle both Unix (\n) and Windows (\r\n) line endings
const lines = result.text.trim().split(/\r?\n/).filter(Boolean)
// Parse JSON lines from ripgrep output
return lines
.map((line) => JSON.parse(line))
.map((parsed) => Result.parse(parsed))
.filter((r) => r.type === "match")
.map((r) => r.data)
}
}

View File

@@ -2,7 +2,7 @@ import { Hono } from "hono"
import { describeRoute, validator, resolver } from "hono-openapi"
import z from "zod"
import { File } from "../../file"
import { Ripgrep } from "../../file/ripgrep"
import { Fff } from "../../file/fff"
import { LSP } from "../../lsp"
import { Instance } from "../../project/instance"
import { lazy } from "../../util/lazy"
@@ -13,14 +13,14 @@ export const FileRoutes = lazy(() =>
"/find",
describeRoute({
summary: "Find text",
description: "Search for text patterns across files in the project using ripgrep.",
description: "Search for text patterns across files in the project.",
operationId: "find.text",
responses: {
200: {
description: "Matches",
content: {
"application/json": {
schema: resolver(Ripgrep.Match.shape.data.array()),
schema: resolver(Fff.Match.array()),
},
},
},
@@ -34,7 +34,7 @@ export const FileRoutes = lazy(() =>
),
async (c) => {
const pattern = c.req.valid("query").pattern
const result = await Ripgrep.search({
const result = await Fff.search({
cwd: Instance.directory,
pattern,
limit: 10,

View File

@@ -1,4 +1,4 @@
import { Ripgrep } from "../file/ripgrep"
import { Fff } from "../file/fff"
import { Instance } from "../project/instance"
@@ -41,7 +41,7 @@ export namespace SystemPrompt {
`<directories>`,
` ${
project.vcs === "git" && false
? await Ripgrep.tree({
? await Fff.tree({
cwd: Instance.directory,
limit: 50,
})

View File

@@ -1,11 +1,95 @@
import z from "zod"
import path from "path"
import { Tool } from "./tool"
import { Filesystem } from "../util/filesystem"
import DESCRIPTION from "./glob.txt"
import { Ripgrep } from "../file/ripgrep"
import { Fff } from "../file/fff"
import { Instance } from "../project/instance"
import { assertExternalDirectory } from "./external-directory"
import { Glob } from "../util/glob"
type Row = {
path: string
rel: string
}
function include(pattern: string) {
const val = pattern.trim().replaceAll("\\", "/")
if (!val) return "*"
const flat = val.replaceAll("**/", "").replaceAll("/**", "/")
const idx = flat.lastIndexOf("/")
if (idx < 0) return flat
const dir = flat.slice(0, idx + 1)
const glob = flat.slice(idx + 1)
if (!glob) return dir
return `${dir} ${glob}`
}
function words(text: string) {
return text.trim().split(/\s+/).filter(Boolean)
}
function norm(text: string) {
return text.replaceAll("\\", "/")
}
function hidden(rel: string) {
return norm(rel).split("/").includes(".git")
}
function broad(pattern: string) {
const val = norm(pattern.trim())
if (!val) return true
if (["*", "**", "**/*", "./**", "./**/*"].includes(val)) return true
return /^(\*\*\/)?\*$/.test(val)
}
function allowed(pattern: string, rel: string) {
if (Glob.match(pattern, rel)) return true
const file = rel.split("/").at(-1) ?? rel
return Glob.match(pattern, file)
}
function pick(items: { path: string; relativePath: string }[]) {
return items
.map((item) => ({
path: item.path,
rel: norm(item.relativePath),
}))
.filter((item) => !hidden(item.rel))
}
function top(rows: Row[]) {
const out = new Map<string, number>()
for (const row of rows) {
const parts = row.rel.split("/")
const key = parts.length < 2 ? "." : parts.slice(0, Math.min(2, parts.length - 1)).join("/") + "/"
out.set(key, (out.get(key) ?? 0) + 1)
}
return Array.from(out.entries())
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
.slice(0, 12)
}
async function scan(pattern: string, dir: string) {
const direct = await Glob.scan(pattern, {
cwd: dir,
absolute: true,
include: "file",
dot: true,
})
const out = direct.length > 0 ? direct : await Glob.scan(`**/${pattern}`, {
cwd: dir,
absolute: true,
include: "file",
dot: true,
})
return out
.map((file) => ({
path: file,
rel: norm(path.relative(dir, file)),
}))
.filter((item) => !hidden(item.rel))
}
export const GlobTool = Tool.define("glob", {
description: DESCRIPTION,
@@ -29,35 +113,60 @@ export const GlobTool = Tool.define("glob", {
},
})
let search = params.path ?? Instance.directory
search = path.isAbsolute(search) ? search : path.resolve(Instance.directory, search)
await assertExternalDirectory(ctx, search, { kind: "directory" })
let dir = params.path ?? Instance.directory
dir = path.isAbsolute(dir) ? dir : path.resolve(Instance.directory, dir)
await assertExternalDirectory(ctx, dir, { kind: "directory" })
const limit = 100
const files = []
let truncated = false
for await (const file of Ripgrep.files({
cwd: search,
glob: [params.pattern],
signal: ctx.abort,
})) {
if (files.length >= limit) {
truncated = true
break
const wide = broad(params.pattern)
const size = wide ? 400 : limit + 1
const first = await Fff.files({
cwd: dir,
query: include(params.pattern),
size,
current: path.join(dir, ".opencode"),
})
let fallback = false
let rows = pick(first.items).filter((row) => allowed(params.pattern, row.rel))
if (!rows.length) {
const list = words(params.pattern)
if (list.length >= 3) {
const short = list.slice(0, 2).join(" ")
const next = await Fff.files({
cwd: dir,
query: include(short),
size,
current: path.join(dir, ".opencode"),
})
rows = pick(next.items).filter((row) => allowed(params.pattern, row.rel))
}
const full = path.resolve(search, file)
const stats = Filesystem.stat(full)?.mtime.getTime() ?? 0
files.push({
path: full,
mtime: stats,
})
}
files.sort((a, b) => b.mtime - a.mtime)
if (!rows.length) {
fallback = true
rows = (await scan(params.pattern, dir)).filter((row) => allowed(params.pattern, row.rel))
}
const truncated = rows.length > limit
const files = rows.slice(0, limit).map((row) => row.path)
const output = []
if (files.length === 0) output.push("No files found")
if (files.length > 0) {
output.push(...files.map((f) => f.path))
output.push(...files)
if (wide && truncated) {
const dirs = top(rows)
if (dirs.length > 0) {
output.push("")
output.push("Top directories in this result set:")
output.push(...dirs.map(([dir, count]) => `${dir} (${count})`))
}
}
if (fallback) {
output.push("")
output.push("(Used filesystem glob fallback for this pattern.)")
}
if (truncated) {
output.push("")
output.push(
@@ -67,7 +176,7 @@ export const GlobTool = Tool.define("glob", {
}
return {
title: path.relative(Instance.worktree, search),
title: path.relative(Instance.worktree, dir),
metadata: {
count: files.length,
truncated,

View File

@@ -1,6 +1,6 @@
- Fast file pattern matching tool that works with any codebase size
- Fast file pattern matching tool that uses fuzzy-first indexing and frecency ranking
- Supports glob patterns like "**/*.js" or "src/**/*.ts"
- Returns matching file paths sorted by modification time
- Returns matching file paths prioritized by recent and relevant files
- Use this tool when you need to find files by name patterns
- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead
- You have the capability to call multiple tools in a single response. It is always better to speculatively perform multiple searches as a batch that are potentially useful.

View File

@@ -1,16 +1,136 @@
import z from "zod"
import { text } from "node:stream/consumers"
import { Tool } from "./tool"
import { Filesystem } from "../util/filesystem"
import { Ripgrep } from "../file/ripgrep"
import { Process } from "../util/process"
import { Fff } from "../file/fff"
import type { GrepMode } from "@ff-labs/fff-node"
import DESCRIPTION from "./grep.txt"
import { Instance } from "../project/instance"
import path from "path"
import { assertExternalDirectory } from "./external-directory"
import { Glob } from "../util/glob"
const MAX_LINE_LENGTH = 2000
const MAX_LINE = 180
const MAX_MATCH = 100
const MAX_DEF_FIRST = 8
const MAX_DEF_NEXT = 5
function isRegex(pattern: string) {
return /[.*+?^${}()|[\]\\]/.test(pattern)
}
function isConstraint(text: string) {
return text.startsWith("!") || text.startsWith("*") || text.endsWith("/")
}
function clean(text: string) {
return text.replaceAll(":", "").replaceAll("-", "").replaceAll("_", "").toLowerCase().trim()
}
function include(text?: string) {
if (!text) return undefined
const val = text.trim().replaceAll("\\", "/")
if (!val) return undefined
const flat = val.replaceAll("**/", "").replaceAll("/**", "/")
const idx = flat.lastIndexOf("/")
if (idx < 0) return flat
const dir = flat.slice(0, idx + 1)
const glob = flat.slice(idx + 1)
if (!glob) return dir
return `${dir} ${glob}`
}
function query(pattern: string, inc?: string) {
if (!inc) return pattern
return `${inc} ${pattern}`.trim()
}
function norm(text: string) {
return text.replaceAll("\\", "/")
}
function allowed(hit: Fff.Hit, inc?: string) {
if (!inc) return true
const rel = norm(hit.relativePath)
if (Glob.match(inc, rel)) return true
return Glob.match(inc, norm(hit.fileName))
}
function def(line: string) {
const text = line.trim()
if (!text) return false
return /^(export\s+)?(default\s+)?(async\s+)?(function|class|interface|type|enum|const|let|var)\b/.test(text)
}
function imp(line: string) {
return /^(import\b|export\s+\{.*\}\s+from\b|use\b|#include\b|require\()/.test(line.trim())
}
function line(text: string, ranges: [number, number][]) {
const trim = text.trim()
if (trim.length <= MAX_LINE) return trim
const first = ranges[0]
if (!first) return trim.slice(0, MAX_LINE - 3) + "..."
const start = Math.max(0, first[0] - Math.floor(MAX_LINE / 3))
const end = Math.min(trim.length, start + MAX_LINE)
const body = trim.slice(start, end)
const pre = start > 0 ? "..." : ""
const post = end < trim.length ? "..." : ""
return pre + body + post
}
function group(rows: Item[]) {
const out = new Map<string, Item[]>()
for (const row of rows) {
const list = out.get(row.hit.path)
if (list) {
list.push(row)
continue
}
out.set(row.hit.path, [row])
}
return out
}
type Item = {
hit: Fff.Hit
def: boolean
imp: boolean
idx: number
}
async function run(input: {
cwd: string
pattern: string
inc?: string
mode: GrepMode
max: number
before: number
after: number
}) {
const first = await Fff.grep({
cwd: input.cwd,
query: query(input.pattern, include(input.inc)),
mode: input.mode,
max: input.max,
before: input.before,
after: input.after,
})
const head = first.items.filter((hit) => allowed(hit, input.inc))
if (head.length) return { out: first, hits: head }
if (!input.inc) return { out: first, hits: head }
const raw = await Fff.grep({
cwd: input.cwd,
query: input.pattern,
mode: input.mode,
max: input.max,
before: input.before,
after: input.after,
})
return {
out: raw,
hits: raw.items.filter((hit) => allowed(hit, input.inc)),
}
}
export const GrepTool = Tool.define("grep", {
description: DESCRIPTION,
@@ -35,122 +155,161 @@ export const GrepTool = Tool.define("grep", {
},
})
let searchPath = params.path ?? Instance.directory
searchPath = path.isAbsolute(searchPath) ? searchPath : path.resolve(Instance.directory, searchPath)
await assertExternalDirectory(ctx, searchPath, { kind: "directory" })
let dir = params.path ?? Instance.directory
dir = path.isAbsolute(dir) ? dir : path.resolve(Instance.directory, dir)
await assertExternalDirectory(ctx, dir, { kind: "directory" })
const rgPath = await Ripgrep.filepath()
const args = ["-nH", "--hidden", "--no-messages", "--field-match-separator=|", "--regexp", params.pattern]
if (params.include) {
args.push("--glob", params.include)
}
args.push(searchPath)
const proc = Process.spawn([rgPath, ...args], {
stdout: "pipe",
stderr: "pipe",
abort: ctx.abort,
const mode = isRegex(params.pattern) ? "regex" : "plain"
const exact = await run({
cwd: dir,
pattern: params.pattern,
inc: params.include,
mode,
max: 10,
before: 0,
after: 4,
})
if (!proc.stdout || !proc.stderr) {
throw new Error("Process output not available")
}
let phase = "exact"
let note = ""
let warn = exact.out.regexFallbackError
let hits = exact.hits
const output = await text(proc.stdout)
const errorOutput = await text(proc.stderr)
const exitCode = await proc.exited
// Exit codes: 0 = matches found, 1 = no matches, 2 = errors (but may still have matches)
// With --no-messages, we suppress error output but still get exit code 2 for broken symlinks etc.
// Only fail if exit code is 2 AND no output was produced
if (exitCode === 1 || (exitCode === 2 && !output.trim())) {
return {
title: params.pattern,
metadata: { matches: 0, truncated: false },
output: "No files found",
}
}
if (exitCode !== 0 && exitCode !== 2) {
throw new Error(`ripgrep failed: ${errorOutput}`)
}
const hasErrors = exitCode === 2
// Handle both Unix (\n) and Windows (\r\n) line endings
const lines = output.trim().split(/\r?\n/)
const matches = []
for (const line of lines) {
if (!line) continue
const [filePath, lineNumStr, ...lineTextParts] = line.split("|")
if (!filePath || !lineNumStr || lineTextParts.length === 0) continue
const lineNum = parseInt(lineNumStr, 10)
const lineText = lineTextParts.join("|")
const stats = Filesystem.stat(filePath)
if (!stats) continue
matches.push({
path: filePath,
modTime: stats.mtime.getTime(),
lineNum,
lineText,
})
}
matches.sort((a, b) => b.modTime - a.modTime)
const limit = 100
const truncated = matches.length > limit
const finalMatches = truncated ? matches.slice(0, limit) : matches
if (finalMatches.length === 0) {
return {
title: params.pattern,
metadata: { matches: 0, truncated: false },
output: "No files found",
}
}
const totalMatches = matches.length
const outputLines = [`Found ${totalMatches} matches${truncated ? ` (showing first ${limit})` : ""}`]
let currentFile = ""
for (const match of finalMatches) {
if (currentFile !== match.path) {
if (currentFile !== "") {
outputLines.push("")
if (!hits.length) {
const words = params.pattern.trim().split(/\s+/).filter(Boolean)
if (words.length >= 2 && !isConstraint(words[0])) {
const next = words.slice(1).join(" ")
const step = await run({
cwd: dir,
pattern: next,
inc: params.include,
mode: isRegex(next) ? "regex" : "plain",
max: 10,
before: 0,
after: 4,
})
warn = warn ?? step.out.regexFallbackError
if (step.hits.length > 0 && step.hits.length <= 10) {
phase = "broad"
note = `0 exact matches. Broadened query \`${next}\`:`
hits = step.hits
}
currentFile = match.path
outputLines.push(`${match.path}:`)
}
const truncatedLineText =
match.lineText.length > MAX_LINE_LENGTH ? match.lineText.substring(0, MAX_LINE_LENGTH) + "..." : match.lineText
outputLines.push(` Line ${match.lineNum}: ${truncatedLineText}`)
}
if (truncated) {
outputLines.push("")
outputLines.push(
`(Results truncated: showing ${limit} of ${totalMatches} matches (${totalMatches - limit} hidden). Consider using a more specific path or pattern.)`,
)
if (!hits.length) {
const fuzzy = clean(params.pattern)
if (fuzzy) {
const step = await run({
cwd: dir,
pattern: fuzzy,
inc: params.include,
mode: "fuzzy",
max: 3,
before: 0,
after: 2,
})
if (step.hits.length) {
phase = "fuzzy"
note = `0 exact matches. ${step.hits.length} approximate:`
hits = step.hits
}
}
}
if (hasErrors) {
outputLines.push("")
outputLines.push("(Some paths were inaccessible and skipped)")
if (!hits.length && params.pattern.includes("/")) {
const files = await Fff.files({
cwd: dir,
query: params.pattern,
size: 1,
})
const row = files.items[0]
const score = files.scores[0]
if (row && score && score.baseScore > params.pattern.length * 10) {
return {
title: params.pattern,
metadata: { matches: 0, truncated: false },
output: `0 content matches. But there is a relevant file path:\n${row.path}`,
}
}
}
if (!hits.length) {
return {
title: params.pattern,
metadata: { matches: 0, truncated: false },
output: "No files found",
}
}
const rows = hits.map((hit, idx) => ({
hit,
idx,
def: def(hit.lineContent),
imp: imp(hit.lineContent),
}))
const hasDef = rows.some((row) => row.def)
const show = hasDef ? rows.filter((row) => !row.imp || row.def) : rows
show.sort((a, b) => {
const ak = a.def ? 0 : a.imp ? 2 : 1
const bk = b.def ? 0 : b.imp ? 2 : 1
if (ak !== bk) return ak - bk
return a.idx - b.idx
})
const total = show.length
const trim = show.slice(0, MAX_MATCH)
const over = total > MAX_MATCH
const files = new Set(trim.map((row) => row.hit.path)).size
const budget = files <= 3 ? 5000 : files <= 8 ? 3500 : 2500
const read = (trim.find((row) => row.def) ?? trim[0]).hit.path
const out: string[] = []
if (phase === "exact") out.push(`Found ${total} matches${over ? ` (showing first ${MAX_MATCH})` : ""}`)
if (phase !== "exact") out.push(note)
out.push(`Read ${read}`)
if (warn) out.push(`! regex failed: ${warn}`)
const by = group(trim)
let used = out.join("\n").length
let cut = false
let firstDef = true
let shown = 0
for (const [file, list] of by.entries()) {
const chunk = ["", `${file}:`]
let add = 0
for (const row of list) {
add++
chunk.push(` Line ${row.hit.lineNumber}: ${line(row.hit.lineContent, row.hit.matchRanges)}`)
if (!row.def) continue
const max = firstDef ? MAX_DEF_FIRST : MAX_DEF_NEXT
firstDef = false
for (const extra of (row.hit.contextAfter ?? []).slice(0, max)) {
chunk.push(` ${line(extra, [])}`)
}
}
const text = chunk.join("\n")
if (used + text.length > budget && shown > 0) {
cut = true
break
}
out.push(...chunk)
used += text.length
shown += add
}
if (over || cut) {
out.push("")
out.push(`(Results truncated: showing first ${shown} results. Consider using a more specific path or pattern.)`)
}
return {
title: params.pattern,
metadata: {
matches: totalMatches,
truncated,
matches: total,
truncated: over || cut,
},
output: outputLines.join("\n"),
output: out.join("\n"),
}
},
})

View File

@@ -1,8 +1,8 @@
- Fast content search tool that works with any codebase size
- Searches file contents using regular expressions
- Supports full regex syntax (eg. "log.*Error", "function\s+\w+", etc.)
- Fast content search tool that uses fuzzy-first indexing and frecency ranking
- Searches file contents with plain text, regex, and typo-tolerant fuzzy fallback
- Supports regex syntax (eg. "log.*Error", "function\s+\w+", etc.)
- Filter files by pattern with the include parameter (eg. "*.js", "*.{ts,tsx}")
- Returns file paths and line numbers with at least one match sorted by modification time
- Returns file paths and line numbers, prioritizing likely definitions and high-signal results
- Includes smart retries (query broadening and path suggestions) when exact matches fail
- Use this tool when you need to find files containing specific patterns
- If you need to identify/count the number of matches within files, use the Bash tool with `rg` (ripgrep) directly. Do NOT use `grep`.
- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead

View File

@@ -3,8 +3,9 @@ import { Tool } from "./tool"
import * as path from "path"
import DESCRIPTION from "./ls.txt"
import { Instance } from "../project/instance"
import { Ripgrep } from "../file/ripgrep"
import { Fff } from "../file/fff"
import { assertExternalDirectory } from "./external-directory"
import { Glob } from "../util/glob"
export const IGNORE_PATTERNS = [
"node_modules/",
@@ -55,11 +56,18 @@ export const ListTool = Tool.define("list", {
})
const ignoreGlobs = IGNORE_PATTERNS.map((p) => `!${p}*`).concat(params.ignore?.map((p) => `!${p}`) || [])
const files = []
for await (const file of Ripgrep.files({ cwd: searchPath, glob: ignoreGlobs, signal: ctx.abort })) {
files.push(file)
if (files.length >= LIMIT) break
}
const rows = (await Glob.scan("**/*", {
cwd: searchPath,
include: "file",
dot: true,
}))
.map((row) => row.replaceAll("\\", "/"))
.filter((row) => {
ctx.abort.throwIfAborted()
return Fff.allowed({ rel: row, glob: ignoreGlobs, hidden: true })
})
.toSorted((a, b) => a.localeCompare(b))
const files = rows.slice(0, LIMIT)
// Build directory structure
const dirs = new Set<string>()
@@ -113,7 +121,7 @@ export const ListTool = Tool.define("list", {
title: path.relative(Instance.worktree, searchPath),
metadata: {
count: files.length,
truncated: files.length >= LIMIT,
truncated: rows.length > LIMIT,
},
output,
}

View File

@@ -3,8 +3,9 @@ import { pathToFileURL } from "url"
import z from "zod"
import { Tool } from "./tool"
import { Skill } from "../skill"
import { Ripgrep } from "../file/ripgrep"
import { Fff } from "../file/fff"
import { iife } from "@/util/iife"
import { Glob } from "../util/glob"
export const SkillTool = Tool.define("skill", async (ctx) => {
const list = await Skill.available(ctx?.agent)
@@ -60,22 +61,17 @@ export const SkillTool = Tool.define("skill", async (ctx) => {
const limit = 10
const files = await iife(async () => {
const arr = []
for await (const file of Ripgrep.files({
ctx.abort.throwIfAborted()
return (await Glob.scan("**/*", {
cwd: dir,
follow: false,
hidden: true,
signal: ctx.abort,
})) {
if (file.includes("SKILL.md")) {
continue
}
arr.push(path.resolve(dir, file))
if (arr.length >= limit) {
break
}
}
return arr
include: "file",
dot: true,
}))
.map((file) => file.replaceAll("\\", "/"))
.filter((file) => Fff.allowed({ rel: file, hidden: true, glob: ["!node_modules/*", "!.git/*"] }))
.filter((file) => !file.includes("SKILL.md"))
.slice(0, limit)
.map((file) => path.resolve(dir, file))
}).then((f) => f.map((file) => `<file>${file}</file>`).join("\n"))
return {

View File

@@ -0,0 +1,52 @@
import { describe, expect, test } from "bun:test"
import fs from "fs/promises"
import path from "path"
import { tmpdir } from "../fixture/fixture"
import { Instance } from "../../src/project/instance"
import { Fff } from "../../src/file/fff"
describe("file.fff", () => {
test("allowed respects hidden filter", async () => {
expect(Fff.allowed({ rel: "visible.txt", hidden: true })).toBe(true)
expect(Fff.allowed({ rel: ".opencode/thing.json", hidden: true })).toBe(true)
expect(Fff.allowed({ rel: ".opencode/thing.json", hidden: false })).toBe(false)
})
test("search returns empty when nothing matches", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "match.ts"), "const value = 'other'\n")
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const hits = await Fff.search({
cwd: tmp.path,
pattern: "needle",
})
expect(hits).toEqual([])
},
})
})
test("tree builds and truncates", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await fs.mkdir(path.join(dir, "a", "b"), { recursive: true })
await Bun.write(path.join(dir, "a", "b", "c.ts"), "export const x = 1\n")
await Bun.write(path.join(dir, "a", "d.ts"), "export const y = 1\n")
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const tree = await Fff.tree({ cwd: tmp.path, limit: 1 })
expect(tree).toContain("a")
expect(tree).toContain("truncated")
},
})
})
})

View File

@@ -1,54 +0,0 @@
import { describe, expect, test } from "bun:test"
import fs from "fs/promises"
import path from "path"
import { tmpdir } from "../fixture/fixture"
import { Ripgrep } from "../../src/file/ripgrep"
describe("file.ripgrep", () => {
test("defaults to include hidden", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "visible.txt"), "hello")
await fs.mkdir(path.join(dir, ".opencode"), { recursive: true })
await Bun.write(path.join(dir, ".opencode", "thing.json"), "{}")
},
})
const files = await Array.fromAsync(Ripgrep.files({ cwd: tmp.path }))
const hasVisible = files.includes("visible.txt")
const hasHidden = files.includes(path.join(".opencode", "thing.json"))
expect(hasVisible).toBe(true)
expect(hasHidden).toBe(true)
})
test("hidden false excludes hidden", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "visible.txt"), "hello")
await fs.mkdir(path.join(dir, ".opencode"), { recursive: true })
await Bun.write(path.join(dir, ".opencode", "thing.json"), "{}")
},
})
const files = await Array.fromAsync(Ripgrep.files({ cwd: tmp.path, hidden: false }))
const hasVisible = files.includes("visible.txt")
const hasHidden = files.includes(path.join(".opencode", "thing.json"))
expect(hasVisible).toBe(true)
expect(hasHidden).toBe(false)
})
test("search returns empty when nothing matches", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "match.ts"), "const value = 'other'\n")
},
})
const hits = await Ripgrep.search({
cwd: tmp.path,
pattern: "needle",
})
expect(hits).toEqual([])
})
})

View File

@@ -0,0 +1,100 @@
import { describe, expect, test } from "bun:test"
import path from "path"
import { GlobTool } from "../../src/tool/glob"
import { Instance } from "../../src/project/instance"
import { tmpdir } from "../fixture/fixture"
import { SessionID, MessageID } from "../../src/session/schema"
const ctx = {
sessionID: SessionID.make("ses_test"),
messageID: MessageID.make(""),
callID: "",
agent: "build",
abort: AbortSignal.any([]),
messages: [],
metadata: () => {},
ask: async () => {},
}
describe("tool.glob", () => {
test("finds files by glob pattern", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "src", "foo.ts"), "export const foo = 1\n")
await Bun.write(path.join(dir, "src", "bar.ts"), "export const bar = 1\n")
await Bun.write(path.join(dir, "src", "baz.js"), "export const baz = 1\n")
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const glob = await GlobTool.init()
const result = await glob.execute(
{
pattern: "*.ts",
path: tmp.path,
},
ctx,
)
expect(result.metadata.count).toBe(2)
expect(result.output).toContain(path.join(tmp.path, "src", "foo.ts"))
expect(result.output).toContain(path.join(tmp.path, "src", "bar.ts"))
},
})
})
test("returns no files found for unmatched patterns", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "src", "foo.ts"), "export const foo = 1\n")
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const glob = await GlobTool.init()
const result = await glob.execute(
{
pattern: "*.py",
path: tmp.path,
},
ctx,
)
expect(result.metadata.count).toBe(0)
expect(result.output).toBe("No files found")
},
})
})
test("falls back for brace glob patterns", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "src", "foo.ts"), "export const foo = 1\n")
await Bun.write(path.join(dir, "src", "bar.js"), "export const bar = 1\n")
await Bun.write(path.join(dir, "src", "baz.py"), "print('baz')\n")
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const glob = await GlobTool.init()
const result = await glob.execute(
{
pattern: "*.{ts,js}",
path: tmp.path,
},
ctx,
)
expect(result.metadata.count).toBe(2)
expect(result.output).toContain(path.join(tmp.path, "src", "foo.ts"))
expect(result.output).toContain(path.join(tmp.path, "src", "bar.js"))
},
})
})
})

View File

@@ -84,28 +84,50 @@ describe("tool.grep", () => {
},
})
})
})
describe("CRLF regex handling", () => {
test("regex correctly splits Unix line endings", () => {
const unixOutput = "file1.txt|1|content1\nfile2.txt|2|content2\nfile3.txt|3|content3"
const lines = unixOutput.trim().split(/\r?\n/)
expect(lines.length).toBe(3)
expect(lines[0]).toBe("file1.txt|1|content1")
expect(lines[2]).toBe("file3.txt|3|content3")
test("broadens multi-word query when exact has no match", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "test.txt"), "upload completed\n")
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const grep = await GrepTool.init()
const result = await grep.execute(
{
pattern: "prepare upload",
path: tmp.path,
},
ctx,
)
expect(result.metadata.matches).toBeGreaterThan(0)
expect(result.output).toContain("Broadened query")
},
})
})
test("regex correctly splits Windows CRLF line endings", () => {
const windowsOutput = "file1.txt|1|content1\r\nfile2.txt|2|content2\r\nfile3.txt|3|content3"
const lines = windowsOutput.trim().split(/\r?\n/)
expect(lines.length).toBe(3)
expect(lines[0]).toBe("file1.txt|1|content1")
expect(lines[2]).toBe("file3.txt|3|content3")
})
test("regex handles mixed line endings", () => {
const mixedOutput = "file1.txt|1|content1\nfile2.txt|2|content2\r\nfile3.txt|3|content3"
const lines = mixedOutput.trim().split(/\r?\n/)
expect(lines.length).toBe(3)
test("suggests path when content has no match", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "src", "server", "auth.ts"), "export const token = 1\n")
},
})
await Instance.provide({
directory: tmp.path,
fn: async () => {
const grep = await GrepTool.init()
const result = await grep.execute(
{
pattern: "src/server/auth.ts",
path: tmp.path,
},
ctx,
)
expect(result.metadata.matches).toBe(0)
expect(result.output).toContain("relevant file path")
},
})
})
})