feat(core): implement Stage 2 security and consistency improvements for web_fetch (#22217)

packages/core/src/utils/fetch.test.ts

@@ -5,7 +5,15 @@
  */
 
 import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';
-import { isPrivateIp, isAddressPrivate, fetchWithTimeout } from './fetch.js';
+import {
+  isPrivateIp,
+  isPrivateIpAsync,
+  isAddressPrivate,
+  fetchWithTimeout,
+} from './fetch.js';
+import * as dnsPromises from 'node:dns/promises';
+import type { LookupAddress, LookupAllOptions } from 'node:dns';
+import ipaddr from 'ipaddr.js';
 
 vi.mock('node:dns/promises', () => ({
   lookup: vi.fn(),
@@ -15,9 +23,25 @@ vi.mock('node:dns/promises', () => ({
 const originalFetch = global.fetch;
 global.fetch = vi.fn();
 
+interface ErrorWithCode extends Error {
+  code?: string;
+}
+
 describe('fetch utils', () => {
   beforeEach(() => {
     vi.clearAllMocks();
+    // Default DNS lookup to return a public IP, or the IP itself if valid
+    vi.mocked(
+      dnsPromises.lookup as (
+        hostname: string,
+        options: LookupAllOptions,
+      ) => Promise<LookupAddress[]>,
+    ).mockImplementation(async (hostname: string) => {
+      if (ipaddr.isValid(hostname)) {
+        return [{ address: hostname, family: hostname.includes(':') ? 6 : 4 }];
+      }
+      return [{ address: '93.184.216.34', family: 4 }];
+    });
   });
 
   afterAll(() => {
@@ -99,6 +123,43 @@ describe('fetch utils', () => {
     });
   });
 
+  describe('isPrivateIpAsync', () => {
+    it('should identify private IPs directly', async () => {
+      expect(await isPrivateIpAsync('http://10.0.0.1/')).toBe(true);
+    });
+
+    it('should identify domains resolving to private IPs', async () => {
+      vi.mocked(
+        dnsPromises.lookup as (
+          hostname: string,
+          options: LookupAllOptions,
+        ) => Promise<LookupAddress[]>,
+      ).mockImplementation(async () => [{ address: '10.0.0.1', family: 4 }]);
+      expect(await isPrivateIpAsync('http://malicious.com/')).toBe(true);
+    });
+
+    it('should identify domains resolving to public IPs as non-private', async () => {
+      vi.mocked(
+        dnsPromises.lookup as (
+          hostname: string,
+          options: LookupAllOptions,
+        ) => Promise<LookupAddress[]>,
+      ).mockImplementation(async () => [{ address: '8.8.8.8', family: 4 }]);
+      expect(await isPrivateIpAsync('http://google.com/')).toBe(false);
+    });
+
+    it('should throw error if DNS resolution fails (fail closed)', async () => {
+      vi.mocked(dnsPromises.lookup).mockRejectedValue(new Error('DNS Error'));
+      await expect(isPrivateIpAsync('http://unreachable.com/')).rejects.toThrow(
+        'Failed to verify if URL resolves to private IP',
+      );
+    });
+
+    it('should return false for invalid URLs instead of throwing verification error', async () => {
+      expect(await isPrivateIpAsync('not-a-url')).toBe(false);
+    });
+  });
+
   describe('fetchWithTimeout', () => {
     it('should handle timeouts', async () => {
       vi.mocked(global.fetch).mockImplementation(
@@ -106,9 +167,10 @@ describe('fetch utils', () => {
           new Promise((_resolve, reject) => {
             if (init?.signal) {
               init.signal.addEventListener('abort', () => {
-                const error = new Error('The operation was aborted');
+                const error = new Error(
+                  'The operation was aborted',
+                ) as ErrorWithCode;
                 error.name = 'AbortError';
-                // @ts-expect-error - for mocking purposes
                 error.code = 'ABORT_ERR';
                 reject(error);
               });

packages/core/src/utils/fetch.ts

@@ -8,6 +8,7 @@ import { getErrorMessage, isNodeError } from './errors.js';
 import { URL } from 'node:url';
 import { Agent, ProxyAgent, setGlobalDispatcher } from 'undici';
 import ipaddr from 'ipaddr.js';
+import { lookup } from 'node:dns/promises';
 
 const DEFAULT_HEADERS_TIMEOUT = 300000; // 5 minutes
 const DEFAULT_BODY_TIMEOUT = 300000; // 5 minutes
@@ -23,6 +24,13 @@ export class FetchError extends Error {
   }
 }
 
+export class PrivateIpError extends Error {
+  constructor(message = 'Access to private network is blocked') {
+    super(message);
+    this.name = 'PrivateIpError';
+  }
+}
+
 // Configure default global dispatcher with higher timeouts
 setGlobalDispatcher(
   new Agent({
@@ -115,6 +123,30 @@ export function isAddressPrivate(address: string): boolean {
   }
 }
 
+/**
+ * Checks if a URL resolves to a private IP address.
+ */
+export async function isPrivateIpAsync(url: string): Promise<boolean> {
+  try {
+    const parsedUrl = new URL(url);
+    const hostname = parsedUrl.hostname;
+
+    if (isLoopbackHost(hostname)) {
+      return false;
+    }
+
+    const addresses = await lookup(hostname, { all: true });
+    return addresses.some((addr) => isAddressPrivate(addr.address));
+  } catch (error) {
+    if (error instanceof TypeError) {
+      return false;
+    }
+    throw new Error('Failed to verify if URL resolves to private IP', {
+      cause: error,
+    });
+  }
+}
+
 /**
  * Creates an undici ProxyAgent that incorporates safe DNS lookup.
  */