Reclassifying Capacity Exhaustion as Terminal Error (#28716)

packages/a2a-server/src/agent/task.ts

@@ -131,9 +131,11 @@ export class Task {
     this.autoExecute = autoExecute;
     this.config.setFallbackModelHandler(
       // For a2a-server, we want to automatically switch to the fallback model
-      // and retry the current request seamlessly. The 'retry_always' intent
-      // achieves this, ensuring a smooth fallback experience for the user.
-      async () => 'retry_always',
+      // for future requests without retrying the current one.
+      async (failedModel, fallbackModel) => {
+        this.config.activateFallbackMode(fallbackModel, failedModel);
+        return 'stop';
+      },
     );
   }
 

packages/core/src/config/config.ts

@@ -1938,6 +1938,9 @@ export class Config implements McpContext, AgentLoopContext {
   }
 
   activateFallbackMode(model: string, failedModel?: string): void {
+    debugLogger.log(
+      `Model fallback activated: switching from ${failedModel ?? 'unknown'} to ${model}`,
+    );
     if (this.getActiveModel() !== model) {
       this.setModel(model, true);
     }

packages/core/src/utils/flashFallback.test.ts

@@ -107,6 +107,36 @@ describe('Retry Utility Fallback Integration', () => {
     expect(mockApiCall).toHaveBeenCalledTimes(3);
   });
 
+  it('should call onPersistent429 immediately on attempt 1 when classifyGoogleError returns TerminalQuotaError', async () => {
+    const mockApiCall = vi
+      .fn()
+      .mockRejectedValue(
+        new TerminalQuotaError('Capacity exhausted', mockGoogleApiError),
+      );
+
+    const mockPersistent429Callback = vi.fn(
+      async () =>
+        // Return null to stop retrying after fallback attempt
+        null,
+    );
+
+    const promise = retryWithBackoff(mockApiCall, {
+      maxAttempts: 10, // High maxAttempts to prove we don't wait for max attempts
+      initialDelayMs: 1,
+      maxDelayMs: 10,
+      onPersistent429: mockPersistent429Callback,
+      authType: AuthType.LOGIN_WITH_GOOGLE,
+    });
+
+    await expect(promise).rejects.toThrow('Capacity exhausted');
+    expect(mockApiCall).toHaveBeenCalledTimes(1); // Only called once because it's terminal and fallback returned null
+    expect(mockPersistent429Callback).toHaveBeenCalledTimes(1);
+    expect(mockPersistent429Callback).toHaveBeenCalledWith(
+      AuthType.LOGIN_WITH_GOOGLE,
+      expect.any(TerminalQuotaError),
+    );
+  });
+
   it('should trigger onPersistent429 when HTTP 499 persists through all retry attempts', async () => {
     let fallbackCalled = false;
     const mockError: HttpError = new Error('Simulated 499 error');

packages/core/src/utils/googleQuotaErrors.test.ts

@@ -81,7 +81,7 @@ describe('classifyGoogleError', () => {
     }
   });
 
-  it('should return RetryableQuotaError with delay for 503 Service Unavailable with RetryInfo', () => {
+  it('should return TerminalQuotaError for MODEL_CAPACITY_EXHAUSTED even with RetryInfo headers', () => {
     const apiError: GoogleApiError = {
       code: 503,
       message:
@@ -103,8 +103,7 @@ describe('classifyGoogleError', () => {
     };
     vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
     const result = classifyGoogleError(new Error());
-    expect(result).toBeInstanceOf(RetryableQuotaError);
-    expect((result as RetryableQuotaError).retryDelayMs).toBe(9000);
+    expect(result).toBeInstanceOf(TerminalQuotaError);
   });
 
   it('should return TerminalQuotaError for MODEL_CAPACITY_EXHAUSTED when no retry delay is specified', () => {
@@ -126,6 +125,24 @@ describe('classifyGoogleError', () => {
     expect(result).toBeInstanceOf(TerminalQuotaError);
   });
 
+  it('should return TerminalQuotaError for structured error with details when message contains capacity exhaustion keywords', () => {
+    const apiError: GoogleApiError = {
+      code: 429,
+      message: 'You have exhausted your capacity on this model.',
+      details: [
+        {
+          '@type': 'type.googleapis.com/google.rpc.Help',
+          links: [
+            { description: 'Learn more', url: 'https://support.google.com' },
+          ],
+        },
+      ],
+    };
+    vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
+    const result = classifyGoogleError(new Error());
+    expect(result).toBeInstanceOf(TerminalQuotaError);
+  });
+
   it('should return TerminalQuotaError for MODEL_CAPACITY_EXHAUSTED even when the domain is not a Cloud Code domain (domain-agnostic)', () => {
     const apiError: GoogleApiError = {
       code: 429,
@@ -396,6 +413,28 @@ describe('classifyGoogleError', () => {
     expect((result as TerminalQuotaError).reason).toBe('RATE_LIMIT_EXCEEDED');
   });
 
+  it('should return TerminalQuotaError for Cloud Code RATE_LIMIT_EXCEEDED without a specified server delay', () => {
+    const apiError: GoogleApiError = {
+      code: 429,
+      message: 'Rate limit exceeded',
+      details: [
+        {
+          '@type': 'type.googleapis.com/google.rpc.ErrorInfo',
+          reason: 'RATE_LIMIT_EXCEEDED',
+          domain: 'cloudcode-pa.googleapis.com',
+          metadata: {
+            uiMessage: 'true',
+            model: 'gemini-2.5-pro',
+          },
+        },
+      ],
+    };
+    vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
+    const result = classifyGoogleError(new Error());
+    expect(result).toBeInstanceOf(TerminalQuotaError);
+    expect((result as TerminalQuotaError).reason).toBe('RATE_LIMIT_EXCEEDED');
+  });
+
   it('should return TerminalQuotaError for Cloud Code QUOTA_EXHAUSTED', () => {
     const apiError: GoogleApiError = {
       code: 429,

packages/core/src/utils/googleQuotaErrors.ts

@@ -289,16 +289,23 @@ export function classifyGoogleError(error: unknown): unknown {
         return new RetryableQuotaError(errorMessage, cause, retryDelaySeconds);
       }
     } else if (status === 429 || status === 499 || status === 503) {
-      // Fallback: If it is a 429, 499, or 503 but doesn't have a specific "retry in" message,
-      // assume it is a temporary rate limit and retry.
-      return new RetryableQuotaError(
-        errorMessage,
-        googleApiError ?? {
-          code: status,
-          message: errorMessage,
-          details: [],
-        },
-      );
+      const cause = googleApiError ?? {
+        code: status,
+        message: errorMessage,
+        details: [],
+      };
+
+      // If the error message indicates capacity exhaustion, classify as TerminalQuotaError
+      if (
+        /exhausted your capacity|capacity exceeded|MODEL_CAPACITY_EXHAUSTED/i.test(
+          errorMessage,
+        )
+      ) {
+        return new TerminalQuotaError(errorMessage, cause);
+      }
+
+      // Fallback: assume it is a temporary rate limit and retry.
+      return new RetryableQuotaError(errorMessage, cause);
     }
 
     return error; // Not a retryable error we can handle with structured details or a parsable retry message.
@@ -338,6 +345,19 @@ export function classifyGoogleError(error: unknown): unknown {
   }
 
   if (errorInfo) {
+    // Always treat capacity exhaustion as terminal error to trigger immediate model fallback
+    if (
+      errorInfo.reason === 'MODEL_CAPACITY_EXHAUSTED' ||
+      errorInfo.reason === 'MODEL_CAPACITY_EXCEEDED'
+    ) {
+      return new TerminalQuotaError(
+        googleApiError.message,
+        googleApiError,
+        delaySeconds,
+        errorInfo.reason,
+      );
+    }
+
     // INSUFFICIENT_G1_CREDITS_BALANCE is always terminal, regardless of domain
     if (errorInfo.reason === 'INSUFFICIENT_G1_CREDITS_BALANCE') {
       return new TerminalQuotaError(
@@ -348,28 +368,19 @@ export function classifyGoogleError(error: unknown): unknown {
       );
     }
 
-    if (
-      errorInfo.reason === 'MODEL_CAPACITY_EXHAUSTED' ||
-      errorInfo.reason === 'MODEL_CAPACITY_EXCEEDED'
-    ) {
-      // If no server backoff delay is specified, treat capacity exhaustion as a terminal error
-      // to trigger immediate model fallback without retrying on the same exhausted model.
-      if (delaySeconds === undefined) {
-        return new TerminalQuotaError(
-          googleApiError.message,
-          googleApiError,
-          delaySeconds,
-          errorInfo.reason,
-        );
-      }
-      // Otherwise, fall through to RetryableQuotaError to honor the server's requested delay.
-    }
-
     // New Cloud Code API quota handling
     if (errorInfo.domain) {
       if (isCloudCodeDomain(errorInfo.domain)) {
         if (errorInfo.reason === 'RATE_LIMIT_EXCEEDED') {
-          const effectiveDelay = delaySeconds ?? 10;
+          if (delaySeconds === undefined) {
+            return new TerminalQuotaError(
+              googleApiError.message,
+              googleApiError,
+              undefined,
+              errorInfo.reason,
+            );
+          }
+          const effectiveDelay = delaySeconds;
           if (effectiveDelay > MAX_RETRYABLE_DELAY_SECONDS) {
             return new TerminalQuotaError(
               googleApiError.message,
@@ -437,6 +448,15 @@ export function classifyGoogleError(error: unknown): unknown {
     }
   }
 
+  // If the error message indicates capacity exhaustion, classify as TerminalQuotaError
+  if (
+    /exhausted your capacity|capacity exceeded|MODEL_CAPACITY_EXHAUSTED/i.test(
+      errorMessage,
+    )
+  ) {
+    return new TerminalQuotaError(errorMessage, googleApiError);
+  }
+
   // If we reached this point, the status is 429, 499, or 503 and we have details,
   // but no specific violation was matched. We return a generic retryable error.
   return new RetryableQuotaError(errorMessage, googleApiError);