feat: detect new files in @ recommendations with watcher based updates (#25256)

packages/core/src/config/config.ts

@@ -616,6 +616,7 @@ export interface ConfigParameters {
   fileFiltering?: {
     respectGitIgnore?: boolean;
     respectGeminiIgnore?: boolean;
+    enableFileWatcher?: boolean;
     enableRecursiveFileSearch?: boolean;
     enableFuzzySearch?: boolean;
     maxFileCount?: number;
@@ -799,6 +800,7 @@ export class Config implements McpContext, AgentLoopContext {
   private readonly fileFiltering: {
     respectGitIgnore: boolean;
     respectGeminiIgnore: boolean;
+    enableFileWatcher: boolean;
     enableRecursiveFileSearch: boolean;
     enableFuzzySearch: boolean;
     maxFileCount: number;
@@ -1080,6 +1082,10 @@ export class Config implements McpContext, AgentLoopContext {
       respectGeminiIgnore:
         params.fileFiltering?.respectGeminiIgnore ??
         DEFAULT_FILE_FILTERING_OPTIONS.respectGeminiIgnore,
+      enableFileWatcher:
+        params.fileFiltering?.enableFileWatcher ??
+        DEFAULT_FILE_FILTERING_OPTIONS.enableFileWatcher ??
+        true,
       enableRecursiveFileSearch:
         params.fileFiltering?.enableRecursiveFileSearch ?? true,
       enableFuzzySearch: params.fileFiltering?.enableFuzzySearch ?? true,
@@ -2831,6 +2837,7 @@ export class Config implements McpContext, AgentLoopContext {
     return {
       respectGitIgnore: this.fileFiltering.respectGitIgnore,
       respectGeminiIgnore: this.fileFiltering.respectGeminiIgnore,
+      enableFileWatcher: this.fileFiltering.enableFileWatcher,
       maxFileCount: this.fileFiltering.maxFileCount,
       searchTimeout: this.fileFiltering.searchTimeout,
       customIgnoreFilePaths: this.fileFiltering.customIgnoreFilePaths,

packages/core/src/config/constants.ts

@@ -7,6 +7,7 @@
 export interface FileFilteringOptions {
   respectGitIgnore: boolean;
   respectGeminiIgnore: boolean;
+  enableFileWatcher?: boolean;
   maxFileCount?: number;
   searchTimeout?: number;
   customIgnoreFilePaths: string[];
@@ -16,6 +17,7 @@ export interface FileFilteringOptions {
 export const DEFAULT_MEMORY_FILE_FILTERING_OPTIONS: FileFilteringOptions = {
   respectGitIgnore: false,
   respectGeminiIgnore: true,
+  enableFileWatcher: false,
   maxFileCount: 20000,
   searchTimeout: 5000,
   customIgnoreFilePaths: [],
@@ -25,6 +27,7 @@ export const DEFAULT_MEMORY_FILE_FILTERING_OPTIONS: FileFilteringOptions = {
 export const DEFAULT_FILE_FILTERING_OPTIONS: FileFilteringOptions = {
   respectGitIgnore: true,
   respectGeminiIgnore: true,
+  enableFileWatcher: false,
   maxFileCount: 20000,
   searchTimeout: 5000,
   customIgnoreFilePaths: [],

packages/core/src/utils/filesearch/fileSearch.test.ts

@@ -6,6 +6,7 @@
 
 import { describe, it, expect, afterEach, vi } from 'vitest';
 import path from 'node:path';
+import fs from 'node:fs/promises';
 import { FileSearchFactory, AbortError, filter } from './fileSearch.js';
 import { createTmpDir, cleanupTmpDir } from '@google/gemini-cli-test-utils';
 import * as crawler from './crawler.js';
@@ -150,6 +151,70 @@ describe('FileSearch', () => {
     ]);
   });
 
+  it('should include newly created directory when watcher is enabled', async () => {
+    tmpDir = await createTmpDir({
+      src: ['main.js'],
+    });
+
+    const fileSearch = FileSearchFactory.create({
+      projectRoot: tmpDir,
+      fileDiscoveryService: new FileDiscoveryService(tmpDir, {
+        respectGitIgnore: false,
+        respectGeminiIgnore: false,
+      }),
+      ignoreDirs: [],
+      cache: false,
+      cacheTtl: 0,
+      enableFileWatcher: true,
+      enableRecursiveFileSearch: true,
+      enableFuzzySearch: true,
+    });
+
+    await fileSearch.initialize();
+    await new Promise((resolve) => setTimeout(resolve, 300));
+    await fs.mkdir(path.join(tmpDir, 'new-folder'));
+    await new Promise((resolve) => setTimeout(resolve, 1200));
+
+    const results = await fileSearch.search('new-folder');
+    expect(results).toContain('new-folder/');
+  });
+
+  it('should include newly created file and remove it after deletion when watcher is enabled', async () => {
+    tmpDir = await createTmpDir({
+      src: ['main.js'],
+    });
+
+    const fileSearch = FileSearchFactory.create({
+      projectRoot: tmpDir,
+      fileDiscoveryService: new FileDiscoveryService(tmpDir, {
+        respectGitIgnore: false,
+        respectGeminiIgnore: false,
+      }),
+      ignoreDirs: [],
+      cache: false,
+      cacheTtl: 0,
+      enableFileWatcher: true,
+      enableRecursiveFileSearch: true,
+      enableFuzzySearch: true,
+    });
+
+    await fileSearch.initialize();
+    await new Promise((resolve) => setTimeout(resolve, 300));
+
+    const filePath = path.join(tmpDir, 'watcher-file.txt');
+    await fs.writeFile(filePath, 'hello');
+    await new Promise((resolve) => setTimeout(resolve, 1200));
+
+    let results = await fileSearch.search('watcher-file');
+    expect(results).toContain('watcher-file.txt');
+
+    await fs.rm(filePath, { force: true });
+    await new Promise((resolve) => setTimeout(resolve, 1200));
+
+    results = await fileSearch.search('watcher-file');
+    expect(results).not.toContain('watcher-file.txt');
+  });
+
   it('should filter results with a search pattern', async () => {
     tmpDir = await createTmpDir({
       src: {

packages/core/src/utils/filesearch/fileSearch.ts

@@ -12,6 +12,8 @@ import { crawl } from './crawler.js';
 import { AsyncFzf, type FzfResultItem } from 'fzf';
 import { unescapePath } from '../paths.js';
 import type { FileDiscoveryService } from '../../services/fileDiscoveryService.js';
+import { FileWatcher, type FileWatcherEvent } from './fileWatcher.js';
+import { debugLogger } from '../debugLogger.js';
 
 // Tiebreaker: Prefers shorter paths.
 const byLengthAsc = (a: { item: string }, b: { item: string }) =>
@@ -57,6 +59,7 @@ export interface FileSearchOptions {
   fileDiscoveryService: FileDiscoveryService;
   cache: boolean;
   cacheTtl: number;
+  enableFileWatcher?: boolean;
   enableRecursiveFileSearch: boolean;
   enableFuzzySearch: boolean;
   maxDepth?: number;
@@ -126,13 +129,16 @@ export interface SearchOptions {
 export interface FileSearch {
   initialize(): Promise<void>;
   search(pattern: string, options?: SearchOptions): Promise<string[]>;
+  close?(): Promise<void>;
 }
 
 class RecursiveFileSearch implements FileSearch {
   private ignore: Ignore | undefined;
   private resultCache: ResultCache | undefined;
-  private allFiles: string[] = [];
+  private allFiles: Set<string> = new Set();
   private fzf: AsyncFzf<string[]> | undefined;
+  private fileWatcher: FileWatcher | undefined;
+  private rebuildTimer: NodeJS.Timeout | undefined;
 
   constructor(private readonly options: FileSearchOptions) {}
 
@@ -142,17 +148,112 @@ class RecursiveFileSearch implements FileSearch {
       this.options.ignoreDirs,
     );
 
-    this.allFiles = await crawl({
-      crawlDirectory: this.options.projectRoot,
-      cwd: this.options.projectRoot,
-      ignore: this.ignore,
-      cache: this.options.cache,
-      cacheTtl: this.options.cacheTtl,
-      maxDepth: this.options.maxDepth,
-      maxFiles: this.options.maxFiles ?? 20000,
-    });
+    this.allFiles = new Set(
+      await crawl({
+        crawlDirectory: this.options.projectRoot,
+        cwd: this.options.projectRoot,
+        ignore: this.ignore,
+        cache: this.options.cache,
+        cacheTtl: this.options.cacheTtl,
+        maxDepth: this.options.maxDepth,
+        maxFiles: this.options.maxFiles ?? 20000,
+      }),
+    );
 
     this.buildResultCache();
+
+    if (this.options.enableFileWatcher) {
+      const directoryFilter = this.ignore.getDirectoryFilter();
+      this.fileWatcher = new FileWatcher(
+        this.options.projectRoot,
+        (event) => this.handleFileWatcherEvent(event),
+        {
+          shouldIgnore: (relativePath) => directoryFilter(`${relativePath}/`),
+          onError(error) {
+            debugLogger.error('File search watcher error: ', error);
+          },
+        },
+      );
+      this.fileWatcher.start();
+    }
+  }
+
+  private scheduleRebuild(): void {
+    if (this.rebuildTimer) {
+      clearTimeout(this.rebuildTimer);
+    }
+
+    this.rebuildTimer = setTimeout(() => {
+      this.rebuildTimer = undefined;
+      this.buildResultCache();
+    }, 150);
+  }
+
+  private handleFileWatcherEvent(event: FileWatcherEvent): void {
+    const normalizedPath = event.relativePath.replaceAll('\\', '/');
+    if (!normalizedPath || normalizedPath === '.') {
+      return;
+    }
+
+    const fileFilter = this.ignore?.getFileFilter();
+    const directoryFilter = this.ignore?.getDirectoryFilter();
+
+    let changed = false;
+    switch (event.eventType) {
+      case 'add': {
+        if (
+          fileFilter?.(normalizedPath) ||
+          this.allFiles.size >= (this.options.maxFiles ?? 20000)
+        ) {
+          return;
+        }
+        const sizeBefore = this.allFiles.size;
+        this.allFiles.add(normalizedPath);
+        changed = this.allFiles.size !== sizeBefore;
+        break;
+      }
+      case 'unlink': {
+        changed = this.allFiles.delete(normalizedPath);
+        break;
+      }
+      case 'addDir': {
+        const directoryPath = normalizedPath.endsWith('/')
+          ? normalizedPath
+          : `${normalizedPath}/`;
+        if (
+          directoryFilter?.(directoryPath) ||
+          this.allFiles.size >= (this.options.maxFiles ?? 20000)
+        ) {
+          return;
+        }
+        const sizeBefore = this.allFiles.size;
+        this.allFiles.add(directoryPath);
+        changed = this.allFiles.size !== sizeBefore;
+        break;
+      }
+      case 'unlinkDir': {
+        const directoryPath = normalizedPath.endsWith('/')
+          ? normalizedPath
+          : `${normalizedPath}/`;
+        const toDelete: string[] = [];
+        for (const file of this.allFiles) {
+          if (file === directoryPath || file.startsWith(directoryPath)) {
+            toDelete.push(file);
+          }
+        }
+        changed = toDelete.length > 0;
+        for (const file of toDelete) {
+          this.allFiles.delete(file);
+        }
+        break;
+      }
+      default:
+        return;
+    }
+
+    if (changed) {
+      this.scheduleRebuild();
+    }
   }
 
   async search(
@@ -222,14 +323,24 @@ class RecursiveFileSearch implements FileSearch {
     return results;
   }
 
+  async close(): Promise<void> {
+    await this.fileWatcher?.close();
+    this.fileWatcher = undefined;
+    if (this.rebuildTimer) {
+      clearTimeout(this.rebuildTimer);
+      this.rebuildTimer = undefined;
+    }
+  }
+
   private buildResultCache(): void {
-    this.resultCache = new ResultCache(this.allFiles);
+    const allFiles = [...this.allFiles];
+    this.resultCache = new ResultCache(allFiles);
     if (this.options.enableFuzzySearch) {
       // The v1 algorithm is much faster since it only looks at the first
       // occurrence of the pattern. We use it for search spaces that have >20k
       // files, because the v2 algorithm is just too slow in those cases.
-      this.fzf = new AsyncFzf(this.allFiles, {
-        fuzzy: this.allFiles.length > 20000 ? 'v1' : 'v2',
+      this.fzf = new AsyncFzf(allFiles, {
+        fuzzy: allFiles.length > 20000 ? 'v1' : 'v2',
         forward: false,
         tiebreakers: [byBasenamePrefix, byMatchPosFromEnd, byLengthAsc],
       });

packages/core/src/utils/filesearch/fileWatcher.test.ts

@@ -0,0 +1,220 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { cleanupTmpDir, createTmpDir } from '@google/gemini-cli-test-utils';
+import { FileWatcher, type FileWatcherEvent } from './fileWatcher.js';
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const waitForEvent = async (
+  events: FileWatcherEvent[],
+  predicate: (event: FileWatcherEvent) => boolean,
+  timeoutMs = 4000,
+) => {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    if (events.some(predicate)) {
+      return;
+    }
+    await sleep(50);
+  }
+  throw new Error('Timed out waiting for watcher event');
+};
+
+describe('FileWatcher', () => {
+  const tmpDirs: string[] = [];
+
+  afterEach(async () => {
+    await Promise.all(tmpDirs.map((dir) => cleanupTmpDir(dir)));
+    tmpDirs.length = 0;
+    vi.restoreAllMocks();
+  });
+
+  it('should emit relative add and unlink events for files', async () => {
+    const tmpDir = await createTmpDir({});
+    tmpDirs.push(tmpDir);
+
+    const events: FileWatcherEvent[] = [];
+    const watcher = new FileWatcher(tmpDir, (event) => {
+      events.push(event);
+    });
+
+    watcher.start();
+    await sleep(500);
+
+    const fileName = 'new-file.txt';
+    const filePath = path.join(tmpDir, fileName);
+
+    await fs.writeFile(filePath, 'hello');
+    await sleep(1200);
+
+    await fs.rm(filePath, { force: true });
+    await sleep(1200);
+
+    await watcher.close();
+
+    expect(events).toContainEqual({ eventType: 'add', relativePath: fileName });
+    expect(events).toContainEqual({
+      eventType: 'unlink',
+      relativePath: fileName,
+    });
+  });
+
+  it('should skip ignored paths', async () => {
+    const tmpDir = await createTmpDir({});
+    tmpDirs.push(tmpDir);
+
+    const events: FileWatcherEvent[] = [];
+    const watcher = new FileWatcher(
+      tmpDir,
+      (event) => {
+        events.push(event);
+      },
+      {
+        shouldIgnore: (relativePath) => relativePath.startsWith('ignored'),
+      },
+    );
+
+    watcher.start();
+    await sleep(500);
+
+    await fs.writeFile(path.join(tmpDir, 'ignored.txt'), 'x');
+    await fs.writeFile(path.join(tmpDir, 'kept.txt'), 'x');
+    await sleep(1200);
+
+    await watcher.close();
+
+    expect(events.some((event) => event.relativePath === 'ignored.txt')).toBe(
+      false,
+    );
+    expect(events).toContainEqual({
+      eventType: 'add',
+      relativePath: 'kept.txt',
+    });
+  });
+
+  it('should emit addDir and unlinkDir events for directories', async () => {
+    const tmpDir = await createTmpDir({});
+    tmpDirs.push(tmpDir);
+
+    const events: FileWatcherEvent[] = [];
+    const watcher = new FileWatcher(tmpDir, (event) => {
+      events.push(event);
+    });
+
+    watcher.start();
+    await sleep(500);
+
+    const dirName = 'new-folder';
+    const dirPath = path.join(tmpDir, dirName);
+
+    await fs.mkdir(dirPath);
+    await waitForEvent(
+      events,
+      (event) => event.eventType === 'addDir' && event.relativePath === dirName,
+    );
+
+    await fs.rm(dirPath, { recursive: true, force: true });
+    await waitForEvent(
+      events,
+      (event) =>
+        event.eventType === 'unlinkDir' && event.relativePath === dirName,
+    );
+
+    await watcher.close();
+  });
+
+  it('should normalize nested paths without leading dot prefix', async () => {
+    const tmpDir = await createTmpDir({});
+    tmpDirs.push(tmpDir);
+
+    const events: FileWatcherEvent[] = [];
+    const watcher = new FileWatcher(tmpDir, (event) => {
+      events.push(event);
+    });
+
+    watcher.start();
+    await sleep(500);
+
+    await fs.mkdir(path.join(tmpDir, 'nested'), { recursive: true });
+    await fs.writeFile(path.join(tmpDir, 'nested', 'file.txt'), 'data');
+
+    await waitForEvent(
+      events,
+      (event) =>
+        event.eventType === 'add' && event.relativePath === 'nested/file.txt',
+    );
+
+    const nestedFileEvent = events.find(
+      (event) =>
+        event.eventType === 'add' && event.relativePath.endsWith('/file.txt'),
+    );
+
+    expect(nestedFileEvent).toBeDefined();
+    expect(nestedFileEvent!.relativePath.startsWith('./')).toBe(false);
+    expect(nestedFileEvent!.relativePath.includes('\\')).toBe(false);
+
+    await watcher.close();
+  });
+
+  it('should not emit new events after stop is called', async () => {
+    const tmpDir = await createTmpDir({});
+    tmpDirs.push(tmpDir);
+
+    const events: FileWatcherEvent[] = [];
+    const watcher = new FileWatcher(tmpDir, (event) => {
+      events.push(event);
+    });
+
+    watcher.start();
+    await sleep(500);
+
+    const beforeStopFile = path.join(tmpDir, 'before-stop.txt');
+    await fs.writeFile(beforeStopFile, 'x');
+    await waitForEvent(
+      events,
+      (event) =>
+        event.eventType === 'add' && event.relativePath === 'before-stop.txt',
+    );
+
+    await watcher.close();
+
+    const afterStopCount = events.length;
+    await fs.writeFile(path.join(tmpDir, 'after-stop.txt'), 'x');
+    await sleep(600);
+
+    expect(events.length).toBe(afterStopCount);
+  });
+
+  it('should be safe to start and stop multiple times', async () => {
+    const tmpDir = await createTmpDir({});
+    tmpDirs.push(tmpDir);
+
+    const events: FileWatcherEvent[] = [];
+    const watcher = new FileWatcher(tmpDir, (event) => {
+      events.push(event);
+    });
+
+    watcher.start();
+    watcher.start();
+    await sleep(500);
+
+    await fs.writeFile(path.join(tmpDir, 'idempotent.txt'), 'x');
+    await waitForEvent(
+      events,
+      (event) =>
+        event.eventType === 'add' && event.relativePath === 'idempotent.txt',
+    );
+
+    await watcher.close();
+    await watcher.close();
+
+    expect(events.length).toBeGreaterThan(0);
+  });
+});

packages/core/src/utils/filesearch/fileWatcher.ts

@@ -0,0 +1,103 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { watch, type FSWatcher } from 'chokidar';
+import path from 'node:path';
+
+export type FileWatcherEvent = {
+  eventType: 'add' | 'unlink' | 'addDir' | 'unlinkDir';
+  relativePath: string;
+};
+
+export type FileWatcherCallback = (event: FileWatcherEvent) => void;
+
+type FileWatcherOptions = {
+  shouldIgnore?: (relativePath: string) => boolean;
+  onError?: (error: unknown) => void;
+};
+
+export class FileWatcher {
+  private watcher: FSWatcher | null = null;
+
+  constructor(
+    private readonly projectRoot: string,
+    private readonly onEvent: FileWatcherCallback,
+    private readonly options: FileWatcherOptions = {},
+  ) {}
+
+  private normalizeRelativePath(filePath: string): string {
+    const relativeOrOriginal = path.isAbsolute(filePath)
+      ? path.relative(this.projectRoot, filePath)
+      : filePath;
+
+    const normalized = relativeOrOriginal.replaceAll('\\', '/');
+    if (normalized === '' || normalized === '.') {
+      return '';
+    }
+    if (normalized.startsWith('./')) {
+      return normalized.slice(2);
+    }
+    return normalized;
+  }
+
+  start(): void {
+    if (this.watcher) {
+      return;
+    }
+
+    this.watcher = watch(this.projectRoot, {
+      cwd: this.projectRoot,
+      ignoreInitial: true,
+      awaitWriteFinish: false,
+      followSymlinks: false,
+      persistent: true,
+      ignored: (filePath: string) => {
+        if (!this.options.shouldIgnore) {
+          return false;
+        }
+        const relativePath = this.normalizeRelativePath(filePath);
+        if (!relativePath) {
+          return false;
+        }
+        return this.options.shouldIgnore(relativePath);
+      },
+    });
+
+    this.watcher
+      .on('add', (relativePath: string) => {
+        this.onEvent({
+          eventType: 'add',
+          relativePath: this.normalizeRelativePath(relativePath),
+        });
+      })
+      .on('unlink', (relativePath: string) => {
+        this.onEvent({
+          eventType: 'unlink',
+          relativePath: this.normalizeRelativePath(relativePath),
+        });
+      })
+      .on('addDir', (relativePath: string) => {
+        this.onEvent({
+          eventType: 'addDir',
+          relativePath: this.normalizeRelativePath(relativePath),
+        });
+      })
+      .on('unlinkDir', (relativePath: string) => {
+        this.onEvent({
+          eventType: 'unlinkDir',
+          relativePath: this.normalizeRelativePath(relativePath),
+        });
+      })
+      .on('error', (error: unknown) => {
+        this.options.onError?.(error);
+      });
+  }
+
+  async close(): Promise<void> {
+    await this.watcher?.close();
+    this.watcher = null;
+  }
+}