Log Model Slash Commands (#9250)

This commit is contained in:
Victor May
2025-09-23 18:06:03 -04:00
committed by GitHub
parent 28c3901513
commit 4bd4cd697b
8 changed files with 128 additions and 31 deletions
@@ -39,7 +39,20 @@ const renderComponent = (
const mockConfig = contextValue const mockConfig = contextValue
? ({ ? ({
// --- Functions used by ModelDialog ---
getModel: vi.fn(() => DEFAULT_GEMINI_MODEL_AUTO), getModel: vi.fn(() => DEFAULT_GEMINI_MODEL_AUTO),
setModel: vi.fn(),
// --- Functions used by ClearcutLogger ---
getUsageStatisticsEnabled: vi.fn(() => true),
getSessionId: vi.fn(() => 'mock-session-id'),
getDebugMode: vi.fn(() => false),
getContentGeneratorConfig: vi.fn(() => ({ authType: 'mock' })),
getUseSmartEdit: vi.fn(() => false),
getUseModelRouter: vi.fn(() => false),
getProxy: vi.fn(() => undefined),
// --- Spread test-specific overrides ---
...contextValue, ...contextValue,
} as Config) } as Config)
: undefined; : undefined;
@@ -132,15 +145,15 @@ describe('<ModelDialog />', () => {
}); });
it('calls config.setModel and onClose when DescriptiveRadioButtonSelect.onSelect is triggered', () => { it('calls config.setModel and onClose when DescriptiveRadioButtonSelect.onSelect is triggered', () => {
const mockSetModel = vi.fn(); const { props, mockConfig } = renderComponent({}, {}); // Pass empty object for contextValue
const { props } = renderComponent({}, { setModel: mockSetModel });
const childOnSelect = mockedSelect.mock.calls[0][0].onSelect; const childOnSelect = mockedSelect.mock.calls[0][0].onSelect;
expect(childOnSelect).toBeDefined(); expect(childOnSelect).toBeDefined();
childOnSelect(DEFAULT_GEMINI_MODEL); childOnSelect(DEFAULT_GEMINI_MODEL);
expect(mockSetModel).toHaveBeenCalledWith(DEFAULT_GEMINI_MODEL); // Assert against the default mock provided by renderComponent
expect(mockConfig?.setModel).toHaveBeenCalledWith(DEFAULT_GEMINI_MODEL);
expect(props.onClose).toHaveBeenCalledTimes(1); expect(props.onClose).toHaveBeenCalledTimes(1);
}); });
@@ -17,6 +17,8 @@ import { useKeypress } from '../hooks/useKeypress.js';
import { theme } from '../semantic-colors.js'; import { theme } from '../semantic-colors.js';
import { DescriptiveRadioButtonSelect } from './shared/DescriptiveRadioButtonSelect.js'; import { DescriptiveRadioButtonSelect } from './shared/DescriptiveRadioButtonSelect.js';
import { ConfigContext } from '../contexts/ConfigContext.js'; import { ConfigContext } from '../contexts/ConfigContext.js';
import { ModelSlashCommandEvent } from '@google/gemini-cli-core/src/telemetry/types.js';
import { logModelSlashCommand } from '@google/gemini-cli-core/src/telemetry/loggers.js';
interface ModelDialogProps { interface ModelDialogProps {
onClose: () => void; onClose: () => void;
@@ -71,6 +73,8 @@ export function ModelDialog({ onClose }: ModelDialogProps): React.JSX.Element {
(model: string) => { (model: string) => {
if (config) { if (config) {
config.setModel(model); config.setModel(model);
const event = new ModelSlashCommandEvent(model);
logModelSlashCommand(config, event);
} }
onClose(); onClose();
}, },
@@ -29,6 +29,7 @@ import type {
ExtensionUninstallEvent, ExtensionUninstallEvent,
ModelRoutingEvent, ModelRoutingEvent,
ExtensionEnableEvent, ExtensionEnableEvent,
ModelSlashCommandEvent,
ExtensionDisableEvent, ExtensionDisableEvent,
} from '../types.js'; } from '../types.js';
import { EventMetadataKey } from './event-metadata-key.js'; import { EventMetadataKey } from './event-metadata-key.js';
@@ -69,6 +70,7 @@ export enum EventNames {
EXTENSION_UNINSTALL = 'extension_uninstall', EXTENSION_UNINSTALL = 'extension_uninstall',
TOOL_OUTPUT_TRUNCATED = 'tool_output_truncated', TOOL_OUTPUT_TRUNCATED = 'tool_output_truncated',
MODEL_ROUTING = 'model_routing', MODEL_ROUTING = 'model_routing',
MODEL_SLASH_COMMAND = 'model_slash_command',
} }
export interface LogResponse { export interface LogResponse {
@@ -994,6 +996,20 @@ export class ClearcutLogger {
this.flushIfNeeded(); this.flushIfNeeded();
} }
logModelSlashCommandEvent(event: ModelSlashCommandEvent): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_MODEL_SLASH_COMMAND,
value: event.model_name,
},
];
this.enqueueLogEvent(
this.createLogEvent(EventNames.MODEL_SLASH_COMMAND, data),
);
this.flushIfNeeded();
}
logExtensionDisableEvent(event: ExtensionDisableEvent): void { logExtensionDisableEvent(event: ExtensionDisableEvent): void {
const data: EventValue[] = [ const data: EventValue[] = [
{ {
@@ -401,4 +401,7 @@ export enum EventMetadataKey {
// Logs the source of the decision. // Logs the source of the decision.
GEMINI_CLI_ROUTING_DECISION_SOURCE = 101, GEMINI_CLI_ROUTING_DECISION_SOURCE = 101,
// Logs an event when the user uses the /model command.
GEMINI_CLI_MODEL_SLASH_COMMAND = 103,
} }
+3
View File
@@ -30,6 +30,7 @@ export const EVENT_CONTENT_RETRY = 'gemini_cli.chat.content_retry';
export const EVENT_CONTENT_RETRY_FAILURE = export const EVENT_CONTENT_RETRY_FAILURE =
'gemini_cli.chat.content_retry_failure'; 'gemini_cli.chat.content_retry_failure';
export const EVENT_FILE_OPERATION = 'gemini_cli.file_operation'; export const EVENT_FILE_OPERATION = 'gemini_cli.file_operation';
export const EVENT_MODEL_SLASH_COMMAND = 'gemini_cli.slash_command.model';
export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count'; export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency'; export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count'; export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count';
@@ -45,3 +46,5 @@ export const EVENT_MODEL_ROUTING = 'gemini_cli.model_routing';
export const METRIC_MODEL_ROUTING_LATENCY = 'gemini_cli.model_routing.latency'; export const METRIC_MODEL_ROUTING_LATENCY = 'gemini_cli.model_routing.latency';
export const METRIC_MODEL_ROUTING_FAILURE_COUNT = export const METRIC_MODEL_ROUTING_FAILURE_COUNT =
'gemini_cli.model_routing.failure.count'; 'gemini_cli.model_routing.failure.count';
export const METRIC_MODEL_SLASH_COMMAND_CALL_COUNT =
'gemini_cli.slash_command.model.call_count';
+25
View File
@@ -32,6 +32,7 @@ import {
EVENT_RIPGREP_FALLBACK, EVENT_RIPGREP_FALLBACK,
EVENT_MODEL_ROUTING, EVENT_MODEL_ROUTING,
EVENT_EXTENSION_INSTALL, EVENT_EXTENSION_INSTALL,
EVENT_MODEL_SLASH_COMMAND,
EVENT_EXTENSION_DISABLE, EVENT_EXTENSION_DISABLE,
} from './constants.js'; } from './constants.js';
import type { import type {
@@ -62,6 +63,7 @@ import type {
ExtensionEnableEvent, ExtensionEnableEvent,
ExtensionUninstallEvent, ExtensionUninstallEvent,
ExtensionInstallEvent, ExtensionInstallEvent,
ModelSlashCommandEvent,
} from './types.js'; } from './types.js';
import { import {
recordApiErrorMetrics, recordApiErrorMetrics,
@@ -74,6 +76,7 @@ import {
recordContentRetry, recordContentRetry,
recordContentRetryFailure, recordContentRetryFailure,
recordModelRoutingMetrics, recordModelRoutingMetrics,
recordModelSlashCommand,
} from './metrics.js'; } from './metrics.js';
import { isTelemetrySdkInitialized } from './sdk.js'; import { isTelemetrySdkInitialized } from './sdk.js';
import type { UiEvent } from './uiTelemetry.js'; import type { UiEvent } from './uiTelemetry.js';
@@ -700,6 +703,28 @@ export function logModelRouting(
recordModelRoutingMetrics(config, event); recordModelRoutingMetrics(config, event);
} }
export function logModelSlashCommand(
config: Config,
event: ModelSlashCommandEvent,
): void {
ClearcutLogger.getInstance(config)?.logModelSlashCommandEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_MODEL_SLASH_COMMAND,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Model slash command. Model: ${event.model_name}`,
attributes,
};
logger.emit(logRecord);
recordModelSlashCommand(config, event);
}
export function logExtensionInstallEvent( export function logExtensionInstallEvent(
config: Config, config: Config,
event: ExtensionInstallEvent, event: ExtensionInstallEvent,
+21 -1
View File
@@ -21,9 +21,10 @@ import {
METRIC_CONTENT_RETRY_FAILURE_COUNT, METRIC_CONTENT_RETRY_FAILURE_COUNT,
METRIC_MODEL_ROUTING_LATENCY, METRIC_MODEL_ROUTING_LATENCY,
METRIC_MODEL_ROUTING_FAILURE_COUNT, METRIC_MODEL_ROUTING_FAILURE_COUNT,
METRIC_MODEL_SLASH_COMMAND_CALL_COUNT,
} from './constants.js'; } from './constants.js';
import type { Config } from '../config/config.js'; import type { Config } from '../config/config.js';
import type { ModelRoutingEvent } from './types.js'; import type { ModelRoutingEvent, ModelSlashCommandEvent } from './types.js';
export enum FileOperation { export enum FileOperation {
CREATE = 'create', CREATE = 'create',
@@ -44,6 +45,7 @@ let contentRetryCounter: Counter | undefined;
let contentRetryFailureCounter: Counter | undefined; let contentRetryFailureCounter: Counter | undefined;
let modelRoutingLatencyHistogram: Histogram | undefined; let modelRoutingLatencyHistogram: Histogram | undefined;
let modelRoutingFailureCounter: Counter | undefined; let modelRoutingFailureCounter: Counter | undefined;
let modelSlashCommandCallCounter: Counter | undefined;
let isMetricsInitialized = false; let isMetricsInitialized = false;
function getCommonAttributes(config: Config): Attributes { function getCommonAttributes(config: Config): Attributes {
@@ -130,6 +132,13 @@ export function initializeMetrics(config: Config): void {
valueType: ValueType.INT, valueType: ValueType.INT,
}, },
); );
modelSlashCommandCallCounter = meter.createCounter(
METRIC_MODEL_SLASH_COMMAND_CALL_COUNT,
{
description: 'Counts model slash command calls.',
valueType: ValueType.INT,
},
);
const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, { const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, {
description: 'Count of CLI sessions started.', description: 'Count of CLI sessions started.',
@@ -287,6 +296,17 @@ export function recordContentRetryFailure(config: Config): void {
contentRetryFailureCounter.add(1, getCommonAttributes(config)); contentRetryFailureCounter.add(1, getCommonAttributes(config));
} }
export function recordModelSlashCommand(
config: Config,
event: ModelSlashCommandEvent,
): void {
if (!modelSlashCommandCallCounter || !isMetricsInitialized) return;
modelSlashCommandCallCounter.add(1, {
...getCommonAttributes(config),
'slash_command.model.model_name': event.model_name,
});
}
export function recordModelRoutingMetrics( export function recordModelRoutingMetrics(
config: Config, config: Config,
event: ModelRoutingEvent, event: ModelRoutingEvent,
+40 -27
View File
@@ -567,33 +567,6 @@ export class ModelRoutingEvent implements BaseTelemetryEvent {
} }
} }
export type TelemetryEvent =
| StartSessionEvent
| EndSessionEvent
| UserPromptEvent
| ToolCallEvent
| ApiRequestEvent
| ApiErrorEvent
| ApiResponseEvent
| FlashFallbackEvent
| LoopDetectedEvent
| LoopDetectionDisabledEvent
| NextSpeakerCheckEvent
| KittySequenceOverflowEvent
| MalformedJsonResponseEvent
| IdeConnectionEvent
| ConversationFinishedEvent
| SlashCommandEvent
| FileOperationEvent
| InvalidChunkEvent
| ContentRetryEvent
| ContentRetryFailureEvent
| ExtensionEnableEvent
| ExtensionInstallEvent
| ExtensionUninstallEvent
| ModelRoutingEvent
| ToolOutputTruncatedEvent;
export class ExtensionInstallEvent implements BaseTelemetryEvent { export class ExtensionInstallEvent implements BaseTelemetryEvent {
'event.name': 'extension_install'; 'event.name': 'extension_install';
'event.timestamp': string; 'event.timestamp': string;
@@ -676,6 +649,46 @@ export class ExtensionEnableEvent implements BaseTelemetryEvent {
} }
} }
export class ModelSlashCommandEvent implements BaseTelemetryEvent {
'event.name': 'model_slash_command';
'event.timestamp': string;
model_name: string;
constructor(model_name: string) {
this['event.name'] = 'model_slash_command';
this['event.timestamp'] = new Date().toISOString();
this.model_name = model_name;
}
}
export type TelemetryEvent =
| StartSessionEvent
| EndSessionEvent
| UserPromptEvent
| ToolCallEvent
| ApiRequestEvent
| ApiErrorEvent
| ApiResponseEvent
| FlashFallbackEvent
| LoopDetectedEvent
| LoopDetectionDisabledEvent
| NextSpeakerCheckEvent
| KittySequenceOverflowEvent
| MalformedJsonResponseEvent
| IdeConnectionEvent
| ConversationFinishedEvent
| SlashCommandEvent
| FileOperationEvent
| InvalidChunkEvent
| ContentRetryEvent
| ContentRetryFailureEvent
| ExtensionEnableEvent
| ExtensionInstallEvent
| ExtensionUninstallEvent
| ModelRoutingEvent
| ToolOutputTruncatedEvent
| ModelSlashCommandEvent;
export class ExtensionDisableEvent implements BaseTelemetryEvent { export class ExtensionDisableEvent implements BaseTelemetryEvent {
'event.name': 'extension_disable'; 'event.name': 'extension_disable';
'event.timestamp': string; 'event.timestamp': string;