diff --git a/eslint.config.js b/eslint.config.js index 86f1f6740b..084c9f7743 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -56,6 +56,7 @@ export default tseslint.config( 'eslint.config.js', '**/coverage/**', 'packages/**/dist/**', + 'tools/**/dist/**', 'bundle/**', 'package/bundle/**', '.integration-tests/**', @@ -80,8 +81,8 @@ export default tseslint.config( }, }, { - // Rules for packages/*/src (TS/TSX) - files: ['packages/*/src/**/*.{ts,tsx}'], + // Rules for packages/*/src and tools/caretaker-agent (TS/TSX) + files: ['packages/*/src/**/*.{ts,tsx}', 'tools/caretaker-agent/**/*.{ts,tsx}'], plugins: { import: importPlugin, }, @@ -284,7 +285,7 @@ export default tseslint.config( }, }, { - files: ['packages/*/src/**/*.test.{ts,tsx}'], + files: ['packages/*/src/**/*.test.{ts,tsx}', 'tools/**/*.test.ts'], plugins: { vitest, }, @@ -410,6 +411,13 @@ export default tseslint.config( '@typescript-eslint/no-require-imports': 'off', }, }, + // Allow console logging for backend services (Cloud Logging) + { + files: ['tools/**/*.ts', 'tools/**/*.test.ts'], + rules: { + 'no-console': 'off', + }, + }, // Prettier config must be last prettierConfig, // extra settings for scripts that we run directly with node diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/.dockerignore b/tools/caretaker-agent/cloudrun/ingestion-service/.dockerignore new file mode 100644 index 0000000000..9bc79ae561 --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/.dockerignore @@ -0,0 +1,12 @@ +node_modules +dist +npm-debug.log +.git +.gitignore +*.py +*.pyc +__pycache__ +requirements.txt +project.toml +**/*.test.ts + diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/Dockerfile b/tools/caretaker-agent/cloudrun/ingestion-service/Dockerfile new file mode 100644 index 0000000000..4ae9178b04 --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/Dockerfile @@ -0,0 +1,9 @@ +FROM node:20-slim +WORKDIR /app +COPY package*.json ./ +RUN npm ci +COPY . . +RUN npm run build +EXPOSE 8080 +CMD ["node", "dist/server.js"] + diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/app.test.ts b/tools/caretaker-agent/cloudrun/ingestion-service/app.test.ts new file mode 100644 index 0000000000..9d1737be5e --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/app.test.ts @@ -0,0 +1,355 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + describe, + it, + expect, + vi, + beforeEach, + beforeAll, + afterAll, +} from 'vitest'; +import request from 'supertest'; +import type { Express } from 'express'; + +const mockPublishMessage = vi.fn(); +const mockTopic = vi.fn().mockReturnValue({ + publishMessage: mockPublishMessage, +}); + +vi.mock('@google-cloud/pubsub', () => ({ + PubSub: vi.fn().mockImplementation(() => ({ + // Bind method to mock version + topic: mockTopic, + })), +})); + +vi.mock('@google-cloud/firestore', () => ({ + Firestore: vi.fn().mockImplementation(() => ({})), +})); + +const mockCreateIssue = vi.fn(); +const mockGetIssueRef = vi.fn(); +const mockGetDoc = vi.fn(); + +vi.mock('./db/issuesStore.js', () => ({ + IssuesStore: vi.fn().mockImplementation(() => ({ + createIssue: mockCreateIssue, + getIssueRef: mockGetIssueRef, + })), +})); + +const mockVerifyGithubSignature = vi.fn(); + +vi.mock('./auth/github.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + verifyGithubSignature: mockVerifyGithubSignature, + }; +}); + +describe('Webhook Server Endpoint', () => { + let app: Express; + + beforeAll(async () => { + vi.stubEnv('PROJECT_ID', 'test-project'); + vi.stubEnv('TOPIC_ID', 'test-topic'); + vi.stubEnv('GITHUB_WEBHOOK_SECRET', 'test-secret'); + vi.stubEnv('FIRESTORE_DATABASE', 'test-db'); + vi.stubEnv('FIRESTORE_COLLECTION', 'test-collection'); + + // Import app after environment variables and mocks are set + const appModule = await import('./app.js'); + app = appModule.app; + + mockGetIssueRef.mockReturnValue({ + get: mockGetDoc, + }); + }); + + afterAll(() => { + vi.unstubAllEnvs(); + }); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should return 200 and health status on root endpoint', async () => { + const res = await request(app).get('/'); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + status: 'healthy', + service: 'caretaker-ingestion-service', + revision: 'local', + }); + }); + + it('should return 401 if signature validation fails', async () => { + mockVerifyGithubSignature.mockReturnValue(false); + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'invalid-sig') + .send({ test: true }); + + expect(res.status).toBe(401); + expect(res.body).toEqual({ status: 'error', message: 'Invalid Signature' }); + }); + + it('should return 400 for invalid JSON payload', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .set('Content-Type', 'application/json') + .send('invalid json'); + + expect(res.status).toBe(400); + expect(res.body).toEqual({ + status: 'error', + message: 'Invalid JSON payload', + }); + }); + + it('should return 413 if payload is too large', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + + const largeBody = 'a'.repeat(1024 * 1024 + 1); + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .set('Content-Type', 'application/json') + .send(largeBody); + + expect(res.status).toBe(413); + expect(res.body).toEqual({ + status: 'error', + message: 'Payload too large', + }); + }); + + it('should return 400 if parsed payload is null or not an object', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .set('Content-Type', 'application/json') + .send('null'); + + expect(res.status).toBe(400); + expect(res.body).toEqual({ + status: 'error', + message: 'Invalid payload structure', + }); + }); + + it('should return 200 ignored for unsupported event types', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'pull_request') + .send({ action: 'opened' }); + + expect(res.status).toBe(200); + expect(res.body.status).toBe('ignored'); + expect(res.body.reason).toContain('unsupported event type'); + }); + + it('should return 400 if required payload fields are missing', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .send({ action: 'opened', issue: { title: 'Test' } }); + + expect(res.status).toBe(400); + expect(res.body).toEqual({ + status: 'error', + message: 'Invalid payload structure', + }); + }); + + it('should return 400 if repository format is invalid', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .send({ + action: 'opened', + issue: { number: 1 }, + repository: { full_name: 'invalid-repo-format' }, + }); + + expect(res.status).toBe(400); + expect(res.body).toEqual({ + status: 'error', + message: 'Invalid payload structure', + }); + }); + + it('should accept the webhook, create the issue, and publish to Pub/Sub', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + mockCreateIssue.mockResolvedValue(true); + mockPublishMessage.mockResolvedValue('mock-msg-123'); + + const payload = { + action: 'opened', + issue: { + number: 1, + title: 'Bugs everywhere', + body: 'Please fix this security bug', + }, + repository: { + full_name: 'google/gemini-cli', + }, + sender: { + login: 'tester', + }, + }; + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .send(payload); + + expect(res.status).toBe(202); + expect(res.body).toEqual({ + status: 'accepted', + message_id: 'mock-msg-123', + }); + + expect(mockCreateIssue).toHaveBeenCalledWith( + 'google', + 'gemini-cli', + 1, + 'Bugs everywhere', + ); + expect(mockPublishMessage).toHaveBeenCalled(); + + // Verify rawBody context wrapping is working + const sentBuffer = mockPublishMessage.mock.calls[0][0].data; + const sentData = JSON.parse(sentBuffer.toString()); + expect(sentData.body).toBe( + '\nPlease fix this security bug\n', + ); + }); + + it('should escape untrusted_context tags in the issue body to prevent injection', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + mockCreateIssue.mockResolvedValue(true); + mockPublishMessage.mockResolvedValue('mock-msg-456'); + + const payload = { + action: 'opened', + issue: { + number: 2, + title: 'Injection test', + body: 'Malicious attempt', + }, + repository: { + full_name: 'google/gemini-cli', + }, + }; + + await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .send(payload); + + const sentBuffer = mockPublishMessage.mock.calls[0][0].data; + const sentData = JSON.parse(sentBuffer.toString()); + expect(sentData.body).toBe( + '\nMalicious \\ attempt\n', + ); + }); + + it('should recover and publish to Pub/Sub on retry if issue is UNTRIAGED', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + mockCreateIssue.mockResolvedValue(false); // document exists + mockGetDoc.mockResolvedValue({ + exists: true, + data: () => ({ status: 'UNTRIAGED' }), + get: (field: string) => (field === 'status' ? 'UNTRIAGED' : undefined), + }); + mockPublishMessage.mockResolvedValue('mock-msg-789'); + + const payload = { + action: 'opened', + issue: { + number: 3, + title: 'Bugs everywhere', + }, + repository: { + full_name: 'google/gemini-cli', + }, + }; + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .send(payload); + + expect(res.status).toBe(202); + expect(res.body).toEqual({ + status: 'accepted', + message_id: 'mock-msg-789', + }); + expect(mockPublishMessage).toHaveBeenCalled(); + }); + + it('should ignore duplicate webhooks if the issue is already past UNTRIAGED', async () => { + mockVerifyGithubSignature.mockReturnValue(true); + mockCreateIssue.mockResolvedValue(false); + mockGetDoc.mockResolvedValue({ + exists: true, + data: () => ({ status: 'TRIAGED' }), + get: (field: string) => (field === 'status' ? 'TRIAGED' : undefined), + }); + + const payload = { + action: 'opened', + issue: { + number: 4, + title: 'Bugs everywhere', + }, + repository: { + full_name: 'google/gemini-cli', + }, + }; + + const res = await request(app) + .post('/webhook') + .set('x-hub-signature-256', 'valid-sig') + .set('x-github-event', 'issues') + .send(payload); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ + status: 'ignored', + reason: 'issue already exists: google/gemini-cli#4', + }); + expect(mockPublishMessage).not.toHaveBeenCalled(); + }); +}); diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/app.ts b/tools/caretaker-agent/cloudrun/ingestion-service/app.ts new file mode 100644 index 0000000000..ac61099c77 --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/app.ts @@ -0,0 +1,190 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import express from 'express'; +import { rateLimit } from 'express-rate-limit'; +import { PubSub } from '@google-cloud/pubsub'; +import dotenv from 'dotenv'; +import { Firestore } from '@google-cloud/firestore'; +import { + verifyGithubSignature, + isGitHubWebhookPayload, +} from './auth/github.js'; +import type { GitHubWebhookPayload } from './auth/github.js'; +import { IssuesStore } from './db/issuesStore.js'; + +dotenv.config(); + +const app = express(); + +function getRequiredEnvVar(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +const projectId = getRequiredEnvVar('PROJECT_ID'); +const topicId = getRequiredEnvVar('TOPIC_ID'); +const githubWebhookSecret = getRequiredEnvVar('GITHUB_WEBHOOK_SECRET'); +const databaseId = getRequiredEnvVar('FIRESTORE_DATABASE'); +const collectionName = getRequiredEnvVar('FIRESTORE_COLLECTION'); + +const pubSubClient = new PubSub({ projectId }); +const topic = pubSubClient.topic(topicId); + +const db = new Firestore({ projectId, databaseId }); +const issuesStore = new IssuesStore(db, collectionName); + +// Middleware: read incoming JSON payloads as raw Buffer bytes +app.use(express.raw({ type: 'application/json', limit: '1mb' })); + +const limiter = rateLimit({ + windowMs: 15 * 60 * 1000, // 15 minutes + max: 100, // Limit each IP to 100 requests per window + standardHeaders: true, + legacyHeaders: false, + message: { + status: 'error', + message: 'Too many requests, please try again later.', + }, +}); + +app.get('/', (req, res) => { + res.json({ + status: 'healthy', + service: process.env.K_SERVICE || 'caretaker-ingestion-service', + revision: process.env.K_REVISION || 'local', + }); +}); + +app.post('/webhook', limiter, async (req, res) => { + const header = req.headers['x-hub-signature-256']; + const signature = Array.isArray(header) ? header[0] : header; + + // Github Authentication + if ( + !req.body || + !verifyGithubSignature(req.body, signature, githubWebhookSecret) + ) { + console.error('Unauthorized: HMAC signature mismatch.'); + return res + .status(401) + .json({ status: 'error', message: 'Invalid Signature' }); + } + + const eventType = req.headers['x-github-event']; + if (eventType !== 'issues') { + return res.status(200).json({ + status: 'ignored', + reason: `unsupported event type: ${eventType}`, + }); + } + + let payload: GitHubWebhookPayload; + try { + const parsed: unknown = JSON.parse(req.body.toString()); + if (!isGitHubWebhookPayload(parsed)) { + return res + .status(400) + .json({ status: 'error', message: 'Invalid payload structure' }); + } + payload = parsed; + } catch { + return res + .status(400) + .json({ status: 'error', message: 'Invalid JSON payload' }); + } + + const action = payload.action; + if (action !== 'opened') { + return res.status(200).json({ + status: 'ignored', + reason: `unsupported action: ${action}`, + }); + } + + const issueNumber = payload.issue.number; + const repository = payload.repository.full_name; + + // Payload preprocessing + const rawBody = payload.issue.body || ''; + const escapedBody = rawBody.replace( + /<\/untrusted_context>/g, + '\\', + ); + const sanitizedBody = `\n${escapedBody}\n`; + + const processedData = { + issue_number: issueNumber, + repository, + sender: payload.sender?.login, + body: sanitizedBody, + title: payload.issue.title, + }; + + const [owner, repo] = repository.split('/'); + const title = processedData.title || ''; + + try { + const created = await issuesStore.createIssue( + owner, + repo, + issueNumber, + title, + ); + + if (!created) { + // If the Firestore document already exists, check its status. + // If it is 'UNTRIAGED', we continue to publish to Pub/Sub + // to recover from previous publish failures. + const issueRef = issuesStore.getIssueRef(owner, repo, issueNumber); + const snapshot = await issueRef.get(); + if (snapshot.get('status') !== 'UNTRIAGED') { + return res.status(200).json({ + status: 'ignored', + reason: `issue already exists: ${repository}#${issueNumber}`, + }); + } + } + + // Publish to Pub/Sub + const dataBuffer = Buffer.from(JSON.stringify(processedData)); + const messageId = await topic.publishMessage({ data: dataBuffer }); + + return res.status(202).json({ status: 'accepted', message_id: messageId }); + } catch (error) { + console.error('Error processing webhook:', error); + const message = error instanceof Error ? error.message : 'Unknown error'; + return res.status(500).json({ status: 'error', message }); + } +}); + +// Global Express error handler for middleware failures (e.g., HTTP 413) +app.use( + ( + err: unknown, + req: express.Request, + res: express.Response, + next: express.NextFunction, + ) => { + if ( + err && + typeof err === 'object' && + 'status' in err && + err.status === 413 + ) { + console.error('Payload too large. Limit is 1mb.'); + return res + .status(413) + .json({ status: 'error', message: 'Payload too large' }); + } + next(err); + }, +); + +export { app }; diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/auth/github.test.ts b/tools/caretaker-agent/cloudrun/ingestion-service/auth/github.test.ts new file mode 100644 index 0000000000..5ad3a684bc --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/auth/github.test.ts @@ -0,0 +1,37 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect } from 'vitest'; +import { verifyGithubSignature } from './github.js'; +import * as crypto from 'node:crypto'; + +describe('verifyGithubSignature', () => { + const secret = 'my-secret'; + const payload = '{"test":true}'; + + it('should return true for a valid signature', () => { + const hmac = crypto.createHmac('sha256', secret); + hmac.update(payload); + const validSignature = 'sha256=' + hmac.digest('hex'); + + const result = verifyGithubSignature(payload, validSignature, secret); + expect(result).toBe(true); + }); + + it('should return false if signatureHeader is missing', () => { + const result = verifyGithubSignature(payload, undefined, secret); + expect(result).toBe(false); + }); + + it('should return false for an invalid signature', () => { + const result = verifyGithubSignature( + payload, + 'sha256=invalid-signature', + secret, + ); + expect(result).toBe(false); + }); +}); diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/auth/github.ts b/tools/caretaker-agent/cloudrun/ingestion-service/auth/github.ts new file mode 100644 index 0000000000..8b8b54af54 --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/auth/github.ts @@ -0,0 +1,134 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as crypto from 'node:crypto'; + +/** + * Subset of the GitHub Webhook Payload for issues events. + * @see https://docs.github.com/en/webhooks/webhook-events-and-payloads#issues + */ +export interface GitHubWebhookPayload { + action: string; + issue: { + body?: string | null; // Can be null if description is empty + number: number; + title?: string; + }; + repository: { + /** Expected format: "owner/repo" (e.g. "google-gemini/gemini-cli") */ + full_name: string; + }; + sender?: { + login?: string; + }; +} + +/** Regular expression matching standard GitHub repository format "owner/repo" */ +const GITHUB_REPO_REGEX = /^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/; + +const GITHUB_SIGNATURE_HEADER_LENGTH = 71; // 'sha256=' (7) + 64 hex chars + +/** + * Verify that the payload was sent from GitHub using HMAC SHA256. + * + * @param payloadBody - The raw body of the request (Buffer or string). + * @param signatureHeader - The value of the X-Hub-Signature-256 header. + * @param secret - The GitHub Webhook secret. + * @returns True if the signature is valid, false otherwise. + * @see https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries + */ +export function verifyGithubSignature( + payloadBody: Buffer | string, + signatureHeader: string | undefined, + secret: string, +): boolean { + if ( + !signatureHeader || + signatureHeader.length !== GITHUB_SIGNATURE_HEADER_LENGTH + ) { + return false; + } + + if (!Buffer.isBuffer(payloadBody) && typeof payloadBody !== 'string') { + return false; + } + + const hmac = crypto.createHmac('sha256', secret); + hmac.update(payloadBody); + const expectedSignature = 'sha256=' + hmac.digest('hex'); + + try { + return crypto.timingSafeEqual( + Buffer.from(expectedSignature), + Buffer.from(signatureHeader), + ); + } catch (error) { + console.error('Error verifying GitHub signature:', error); + return false; + } +} + +/** + * Type guard to verify that an unknown object conforms to the GitHubWebhookPayload structure. + * + * @param obj - The object to validate. + * @returns True if the object matches the schema, false otherwise. + */ +export function isGitHubWebhookPayload( + obj: unknown, +): obj is GitHubWebhookPayload { + if (typeof obj !== 'object' || obj === null) { + return false; + } + + const o = obj as GitHubWebhookPayload; + + // 1. Validate 'action' + if (typeof o.action !== 'string') { + return false; + } + + // 2. Validate 'issue' + if (typeof o.issue !== 'object' || o.issue === null) { + return false; + } + if (typeof o.issue.number !== 'number') { + return false; + } + if ( + o.issue.body !== undefined && + o.issue.body !== null && + typeof o.issue.body !== 'string' + ) { + return false; + } + if (o.issue.title !== undefined && typeof o.issue.title !== 'string') { + return false; + } + + // 3. Validate 'repository' + if (typeof o.repository !== 'object' || o.repository === null) { + return false; + } + if (typeof o.repository.full_name !== 'string') { + return false; + } + if (!GITHUB_REPO_REGEX.test(o.repository.full_name)) { + return false; + } + + // 4. Validate 'sender' (optional) + if (o.sender !== undefined) { + if (typeof o.sender !== 'object' || o.sender === null) { + return false; + } + if (o.sender.login !== undefined && typeof o.sender.login !== 'string') { + return false; + } + } + + return true; +} diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/db/issuesStore.test.ts b/tools/caretaker-agent/cloudrun/ingestion-service/db/issuesStore.test.ts new file mode 100644 index 0000000000..86ac17ea24 --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/db/issuesStore.test.ts @@ -0,0 +1,83 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { Mock } from 'vitest'; +import { IssuesStore } from './issuesStore.js'; +import type { Firestore, Transaction } from '@google-cloud/firestore'; + +describe('IssuesStore', () => { + let mockTransaction: { + get: Mock; + set: Mock; + }; + let mockDb: Firestore; + let store: IssuesStore; + + beforeEach(() => { + // Assign mock read/write methods for transaction + mockTransaction = { + get: vi.fn(), + set: vi.fn(), + }; + + // Mock Firestore client + mockDb = { + collection: vi.fn().mockReturnThis(), + doc: vi.fn().mockReturnValue({}), + runTransaction: vi + .fn() + .mockImplementation((callback: (tx: Transaction) => Promise) => + callback(mockTransaction as unknown as Transaction), + ), + } as unknown as Firestore; + + store = new IssuesStore(mockDb, 'issues-collection'); + }); + + it('should initialize a new issue if it does not exist', async () => { + // The transaction should mock that the document does not exist + mockTransaction.get.mockResolvedValue({ exists: false }); + + const result = await store.createIssue( + 'google', + 'gemini-cli', + 123, + 'Test Title', + ); + + expect(result).toBe(true); + expect(mockTransaction.get).toHaveBeenCalled(); + expect(mockTransaction.set).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + status: 'UNTRIAGED', + github_metadata: expect.objectContaining({ + owner: 'google', + repo: 'gemini-cli', + issue_number: 123, + title: 'Test Title', + }), + }), + ); + }); + + it('should return false and skip creation if the issue already exists', async () => { + // The transaction should mock that the document already exists + mockTransaction.get.mockResolvedValue({ exists: true }); + + const result = await store.createIssue( + 'google', + 'gemini-cli', + 123, + 'Test Title', + ); + + expect(result).toBe(false); + expect(mockTransaction.get).toHaveBeenCalled(); + expect(mockTransaction.set).not.toHaveBeenCalled(); + }); +}); diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/db/issuesStore.ts b/tools/caretaker-agent/cloudrun/ingestion-service/db/issuesStore.ts new file mode 100644 index 0000000000..b981cf511f --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/db/issuesStore.ts @@ -0,0 +1,107 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { FieldValue } from '@google-cloud/firestore'; +import type { + Firestore, + DocumentReference, + Transaction, + Timestamp, +} from '@google-cloud/firestore'; + +export type IssueStatus = + | 'UNTRIAGED' + | 'TRIAGING' + | 'NEEDS_INFO' + | 'TRIAGED' + | 'NEEDS_HUMAN' + | 'LOW_QUALITY'; + +export interface IssueDocument { + status: IssueStatus; + triage_attempts: number; + // The ingestion layer does not enforce the schema of workable_spec + workable_spec: Record; + lock: { + holder: string | null; + expires_at: Timestamp | FieldValue | null; + }; + created_at: Timestamp | FieldValue; + updated_at: Timestamp | FieldValue; + github_metadata: { + owner: string; + repo: string; + issue_number: number; + title: string; + }; +} + +export class IssuesStore { + private readonly db: Firestore; + private readonly collectionName: string; + + constructor(db: Firestore, collectionName: string) { + this.db = db; + this.collectionName = collectionName; + } + + // Generates the standardized Firestore document reference for an issue + getIssueRef( + owner: string, + repo: string, + issueNumber: number, + ): DocumentReference { + const docId = `github_${owner}_${repo}_${issueNumber}`; + return this.db.collection(this.collectionName).doc(docId); + } + + // Initializes a new issue document in a transaction + async createIssue( + owner: string, + repo: string, + issueNumber: number, + title: string, + ): Promise { + const docRef = this.getIssueRef(owner, repo, issueNumber); + + try { + return await this.db.runTransaction(async (transaction: Transaction) => { + const snapshot = await transaction.get(docRef); + + if (!snapshot.exists) { + const newIssue: IssueDocument = { + status: 'UNTRIAGED', + triage_attempts: 0, + workable_spec: {}, + lock: { + holder: null, + expires_at: null, + }, + created_at: FieldValue.serverTimestamp(), + updated_at: FieldValue.serverTimestamp(), + github_metadata: { + owner, + repo, + issue_number: issueNumber, + title, + }, + }; + + transaction.set(docRef, newIssue); + return true; + } + return false; + }); + } catch (error) { + console.error( + 'Firestore transaction failed for issue:', + `${owner}/${repo}#${issueNumber}`, + error, + ); + throw error; + } + } +} diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/package.json b/tools/caretaker-agent/cloudrun/ingestion-service/package.json new file mode 100644 index 0000000000..bcf0dae9ba --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/package.json @@ -0,0 +1,28 @@ +{ + "name": "ingestion-service", + "version": "1.0.0", + "description": "Ingestion service for triage worker", + "main": "server.ts", + "scripts": { + "dev": "tsx watch server.ts", + "build": "tsc", + "start": "node dist/server.js", + "test": "vitest run" + }, + "dependencies": { + "@google-cloud/firestore": "^7.7.0", + "@google-cloud/pubsub": "^4.4.0", + "dotenv": "^16.4.5", + "express": "^4.19.2", + "express-rate-limit": "^7.2.0" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.12.12", + "@types/supertest": "^6.0.3", + "supertest": "^7.1.4", + "tsx": "^4.9.3", + "typescript": "^5.4.5", + "vitest": "^1.6.0" + } +} diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/server.ts b/tools/caretaker-agent/cloudrun/ingestion-service/server.ts new file mode 100644 index 0000000000..5fc538014f --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/server.ts @@ -0,0 +1,12 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { app } from './app.js'; + +const port = parseInt(process.env.PORT || '8080', 10); +app.listen(port, '0.0.0.0', () => { + console.log(`Server listening on port ${port}`); +}); diff --git a/tools/caretaker-agent/cloudrun/ingestion-service/tsconfig.json b/tools/caretaker-agent/cloudrun/ingestion-service/tsconfig.json new file mode 100644 index 0000000000..901bd5ccbc --- /dev/null +++ b/tools/caretaker-agent/cloudrun/ingestion-service/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["**/*.ts"], + "exclude": ["node_modules", "dist"] +}