mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-07-04 07:07:16 -07:00
feat(caretaker): implement Cloud Run webhook ingestion service (#28015)
Co-authored-by: Christian Gunderman <gundermanc@google.com>
This commit is contained in:
Chad
+11
-3
@@ -56,6 +56,7 @@ export default tseslint.config(
|
||||
'eslint.config.js',
|
||||
'**/coverage/**',
|
||||
'packages/**/dist/**',
|
||||
'tools/**/dist/**',
|
||||
'bundle/**',
|
||||
'package/bundle/**',
|
||||
'.integration-tests/**',
|
||||
@@ -80,8 +81,8 @@ export default tseslint.config(
|
||||
},
|
||||
},
|
||||
{
|
||||
// Rules for packages/*/src (TS/TSX)
|
||||
files: ['packages/*/src/**/*.{ts,tsx}'],
|
||||
// Rules for packages/*/src and tools/caretaker-agent (TS/TSX)
|
||||
files: ['packages/*/src/**/*.{ts,tsx}', 'tools/caretaker-agent/**/*.{ts,tsx}'],
|
||||
plugins: {
|
||||
import: importPlugin,
|
||||
},
|
||||
@@ -284,7 +285,7 @@ export default tseslint.config(
|
||||
},
|
||||
},
|
||||
{
|
||||
files: ['packages/*/src/**/*.test.{ts,tsx}'],
|
||||
files: ['packages/*/src/**/*.test.{ts,tsx}', 'tools/**/*.test.ts'],
|
||||
plugins: {
|
||||
vitest,
|
||||
},
|
||||
@@ -410,6 +411,13 @@ export default tseslint.config(
|
||||
'@typescript-eslint/no-require-imports': 'off',
|
||||
},
|
||||
},
|
||||
// Allow console logging for backend services (Cloud Logging)
|
||||
{
|
||||
files: ['tools/**/*.ts', 'tools/**/*.test.ts'],
|
||||
rules: {
|
||||
'no-console': 'off',
|
||||
},
|
||||
},
|
||||
// Prettier config must be last
|
||||
prettierConfig,
|
||||
// extra settings for scripts that we run directly with node
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
node_modules
|
||||
dist
|
||||
npm-debug.log
|
||||
.git
|
||||
.gitignore
|
||||
*.py
|
||||
*.pyc
|
||||
__pycache__
|
||||
requirements.txt
|
||||
project.toml
|
||||
**/*.test.ts
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM node:20-slim
|
||||
WORKDIR /app
|
||||
COPY package*.json ./
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
EXPOSE 8080
|
||||
CMD ["node", "dist/server.js"]
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import {
|
||||
describe,
|
||||
it,
|
||||
expect,
|
||||
vi,
|
||||
beforeEach,
|
||||
beforeAll,
|
||||
afterAll,
|
||||
} from 'vitest';
|
||||
import request from 'supertest';
|
||||
import type { Express } from 'express';
|
||||
|
||||
const mockPublishMessage = vi.fn();
|
||||
const mockTopic = vi.fn().mockReturnValue({
|
||||
publishMessage: mockPublishMessage,
|
||||
});
|
||||
|
||||
vi.mock('@google-cloud/pubsub', () => ({
|
||||
PubSub: vi.fn().mockImplementation(() => ({
|
||||
// Bind method to mock version
|
||||
topic: mockTopic,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock('@google-cloud/firestore', () => ({
|
||||
Firestore: vi.fn().mockImplementation(() => ({})),
|
||||
}));
|
||||
|
||||
const mockCreateIssue = vi.fn();
|
||||
const mockGetIssueRef = vi.fn();
|
||||
const mockGetDoc = vi.fn();
|
||||
|
||||
vi.mock('./db/issuesStore.js', () => ({
|
||||
IssuesStore: vi.fn().mockImplementation(() => ({
|
||||
createIssue: mockCreateIssue,
|
||||
getIssueRef: mockGetIssueRef,
|
||||
})),
|
||||
}));
|
||||
|
||||
const mockVerifyGithubSignature = vi.fn();
|
||||
|
||||
vi.mock('./auth/github.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('./auth/github.js')>();
|
||||
return {
|
||||
...actual,
|
||||
verifyGithubSignature: mockVerifyGithubSignature,
|
||||
};
|
||||
});
|
||||
|
||||
describe('Webhook Server Endpoint', () => {
|
||||
let app: Express;
|
||||
|
||||
beforeAll(async () => {
|
||||
vi.stubEnv('PROJECT_ID', 'test-project');
|
||||
vi.stubEnv('TOPIC_ID', 'test-topic');
|
||||
vi.stubEnv('GITHUB_WEBHOOK_SECRET', 'test-secret');
|
||||
vi.stubEnv('FIRESTORE_DATABASE', 'test-db');
|
||||
vi.stubEnv('FIRESTORE_COLLECTION', 'test-collection');
|
||||
|
||||
// Import app after environment variables and mocks are set
|
||||
const appModule = await import('./app.js');
|
||||
app = appModule.app;
|
||||
|
||||
mockGetIssueRef.mockReturnValue({
|
||||
get: mockGetDoc,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should return 200 and health status on root endpoint', async () => {
|
||||
const res = await request(app).get('/');
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({
|
||||
status: 'healthy',
|
||||
service: 'caretaker-ingestion-service',
|
||||
revision: 'local',
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 401 if signature validation fails', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(false);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'invalid-sig')
|
||||
.send({ test: true });
|
||||
|
||||
expect(res.status).toBe(401);
|
||||
expect(res.body).toEqual({ status: 'error', message: 'Invalid Signature' });
|
||||
});
|
||||
|
||||
it('should return 400 for invalid JSON payload', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.set('Content-Type', 'application/json')
|
||||
.send('invalid json');
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body).toEqual({
|
||||
status: 'error',
|
||||
message: 'Invalid JSON payload',
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 413 if payload is too large', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
|
||||
const largeBody = 'a'.repeat(1024 * 1024 + 1);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.set('Content-Type', 'application/json')
|
||||
.send(largeBody);
|
||||
|
||||
expect(res.status).toBe(413);
|
||||
expect(res.body).toEqual({
|
||||
status: 'error',
|
||||
message: 'Payload too large',
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 400 if parsed payload is null or not an object', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.set('Content-Type', 'application/json')
|
||||
.send('null');
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body).toEqual({
|
||||
status: 'error',
|
||||
message: 'Invalid payload structure',
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 200 ignored for unsupported event types', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'pull_request')
|
||||
.send({ action: 'opened' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.status).toBe('ignored');
|
||||
expect(res.body.reason).toContain('unsupported event type');
|
||||
});
|
||||
|
||||
it('should return 400 if required payload fields are missing', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.send({ action: 'opened', issue: { title: 'Test' } });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body).toEqual({
|
||||
status: 'error',
|
||||
message: 'Invalid payload structure',
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 400 if repository format is invalid', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.send({
|
||||
action: 'opened',
|
||||
issue: { number: 1 },
|
||||
repository: { full_name: 'invalid-repo-format' },
|
||||
});
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body).toEqual({
|
||||
status: 'error',
|
||||
message: 'Invalid payload structure',
|
||||
});
|
||||
});
|
||||
|
||||
it('should accept the webhook, create the issue, and publish to Pub/Sub', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
mockCreateIssue.mockResolvedValue(true);
|
||||
mockPublishMessage.mockResolvedValue('mock-msg-123');
|
||||
|
||||
const payload = {
|
||||
action: 'opened',
|
||||
issue: {
|
||||
number: 1,
|
||||
title: 'Bugs everywhere',
|
||||
body: 'Please fix this security bug',
|
||||
},
|
||||
repository: {
|
||||
full_name: 'google/gemini-cli',
|
||||
},
|
||||
sender: {
|
||||
login: 'tester',
|
||||
},
|
||||
};
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.send(payload);
|
||||
|
||||
expect(res.status).toBe(202);
|
||||
expect(res.body).toEqual({
|
||||
status: 'accepted',
|
||||
message_id: 'mock-msg-123',
|
||||
});
|
||||
|
||||
expect(mockCreateIssue).toHaveBeenCalledWith(
|
||||
'google',
|
||||
'gemini-cli',
|
||||
1,
|
||||
'Bugs everywhere',
|
||||
);
|
||||
expect(mockPublishMessage).toHaveBeenCalled();
|
||||
|
||||
// Verify rawBody context wrapping is working
|
||||
const sentBuffer = mockPublishMessage.mock.calls[0][0].data;
|
||||
const sentData = JSON.parse(sentBuffer.toString());
|
||||
expect(sentData.body).toBe(
|
||||
'<untrusted_context>\nPlease fix this security bug\n</untrusted_context>',
|
||||
);
|
||||
});
|
||||
|
||||
it('should escape untrusted_context tags in the issue body to prevent injection', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
mockCreateIssue.mockResolvedValue(true);
|
||||
mockPublishMessage.mockResolvedValue('mock-msg-456');
|
||||
|
||||
const payload = {
|
||||
action: 'opened',
|
||||
issue: {
|
||||
number: 2,
|
||||
title: 'Injection test',
|
||||
body: 'Malicious </untrusted_context> attempt',
|
||||
},
|
||||
repository: {
|
||||
full_name: 'google/gemini-cli',
|
||||
},
|
||||
};
|
||||
|
||||
await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.send(payload);
|
||||
|
||||
const sentBuffer = mockPublishMessage.mock.calls[0][0].data;
|
||||
const sentData = JSON.parse(sentBuffer.toString());
|
||||
expect(sentData.body).toBe(
|
||||
'<untrusted_context>\nMalicious \\</untrusted_context> attempt\n</untrusted_context>',
|
||||
);
|
||||
});
|
||||
|
||||
it('should recover and publish to Pub/Sub on retry if issue is UNTRIAGED', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
mockCreateIssue.mockResolvedValue(false); // document exists
|
||||
mockGetDoc.mockResolvedValue({
|
||||
exists: true,
|
||||
data: () => ({ status: 'UNTRIAGED' }),
|
||||
get: (field: string) => (field === 'status' ? 'UNTRIAGED' : undefined),
|
||||
});
|
||||
mockPublishMessage.mockResolvedValue('mock-msg-789');
|
||||
|
||||
const payload = {
|
||||
action: 'opened',
|
||||
issue: {
|
||||
number: 3,
|
||||
title: 'Bugs everywhere',
|
||||
},
|
||||
repository: {
|
||||
full_name: 'google/gemini-cli',
|
||||
},
|
||||
};
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.send(payload);
|
||||
|
||||
expect(res.status).toBe(202);
|
||||
expect(res.body).toEqual({
|
||||
status: 'accepted',
|
||||
message_id: 'mock-msg-789',
|
||||
});
|
||||
expect(mockPublishMessage).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should ignore duplicate webhooks if the issue is already past UNTRIAGED', async () => {
|
||||
mockVerifyGithubSignature.mockReturnValue(true);
|
||||
mockCreateIssue.mockResolvedValue(false);
|
||||
mockGetDoc.mockResolvedValue({
|
||||
exists: true,
|
||||
data: () => ({ status: 'TRIAGED' }),
|
||||
get: (field: string) => (field === 'status' ? 'TRIAGED' : undefined),
|
||||
});
|
||||
|
||||
const payload = {
|
||||
action: 'opened',
|
||||
issue: {
|
||||
number: 4,
|
||||
title: 'Bugs everywhere',
|
||||
},
|
||||
repository: {
|
||||
full_name: 'google/gemini-cli',
|
||||
},
|
||||
};
|
||||
|
||||
const res = await request(app)
|
||||
.post('/webhook')
|
||||
.set('x-hub-signature-256', 'valid-sig')
|
||||
.set('x-github-event', 'issues')
|
||||
.send(payload);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({
|
||||
status: 'ignored',
|
||||
reason: 'issue already exists: google/gemini-cli#4',
|
||||
});
|
||||
expect(mockPublishMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,190 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import express from 'express';
|
||||
import { rateLimit } from 'express-rate-limit';
|
||||
import { PubSub } from '@google-cloud/pubsub';
|
||||
import dotenv from 'dotenv';
|
||||
import { Firestore } from '@google-cloud/firestore';
|
||||
import {
|
||||
verifyGithubSignature,
|
||||
isGitHubWebhookPayload,
|
||||
} from './auth/github.js';
|
||||
import type { GitHubWebhookPayload } from './auth/github.js';
|
||||
import { IssuesStore } from './db/issuesStore.js';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const app = express();
|
||||
|
||||
function getRequiredEnvVar(name: string): string {
|
||||
const value = process.env[name];
|
||||
if (!value) {
|
||||
throw new Error(`Missing required environment variable: ${name}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
const projectId = getRequiredEnvVar('PROJECT_ID');
|
||||
const topicId = getRequiredEnvVar('TOPIC_ID');
|
||||
const githubWebhookSecret = getRequiredEnvVar('GITHUB_WEBHOOK_SECRET');
|
||||
const databaseId = getRequiredEnvVar('FIRESTORE_DATABASE');
|
||||
const collectionName = getRequiredEnvVar('FIRESTORE_COLLECTION');
|
||||
|
||||
const pubSubClient = new PubSub({ projectId });
|
||||
const topic = pubSubClient.topic(topicId);
|
||||
|
||||
const db = new Firestore({ projectId, databaseId });
|
||||
const issuesStore = new IssuesStore(db, collectionName);
|
||||
|
||||
// Middleware: read incoming JSON payloads as raw Buffer bytes
|
||||
app.use(express.raw({ type: 'application/json', limit: '1mb' }));
|
||||
|
||||
const limiter = rateLimit({
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
max: 100, // Limit each IP to 100 requests per window
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
message: {
|
||||
status: 'error',
|
||||
message: 'Too many requests, please try again later.',
|
||||
},
|
||||
});
|
||||
|
||||
app.get('/', (req, res) => {
|
||||
res.json({
|
||||
status: 'healthy',
|
||||
service: process.env.K_SERVICE || 'caretaker-ingestion-service',
|
||||
revision: process.env.K_REVISION || 'local',
|
||||
});
|
||||
});
|
||||
|
||||
app.post('/webhook', limiter, async (req, res) => {
|
||||
const header = req.headers['x-hub-signature-256'];
|
||||
const signature = Array.isArray(header) ? header[0] : header;
|
||||
|
||||
// Github Authentication
|
||||
if (
|
||||
!req.body ||
|
||||
!verifyGithubSignature(req.body, signature, githubWebhookSecret)
|
||||
) {
|
||||
console.error('Unauthorized: HMAC signature mismatch.');
|
||||
return res
|
||||
.status(401)
|
||||
.json({ status: 'error', message: 'Invalid Signature' });
|
||||
}
|
||||
|
||||
const eventType = req.headers['x-github-event'];
|
||||
if (eventType !== 'issues') {
|
||||
return res.status(200).json({
|
||||
status: 'ignored',
|
||||
reason: `unsupported event type: ${eventType}`,
|
||||
});
|
||||
}
|
||||
|
||||
let payload: GitHubWebhookPayload;
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(req.body.toString());
|
||||
if (!isGitHubWebhookPayload(parsed)) {
|
||||
return res
|
||||
.status(400)
|
||||
.json({ status: 'error', message: 'Invalid payload structure' });
|
||||
}
|
||||
payload = parsed;
|
||||
} catch {
|
||||
return res
|
||||
.status(400)
|
||||
.json({ status: 'error', message: 'Invalid JSON payload' });
|
||||
}
|
||||
|
||||
const action = payload.action;
|
||||
if (action !== 'opened') {
|
||||
return res.status(200).json({
|
||||
status: 'ignored',
|
||||
reason: `unsupported action: ${action}`,
|
||||
});
|
||||
}
|
||||
|
||||
const issueNumber = payload.issue.number;
|
||||
const repository = payload.repository.full_name;
|
||||
|
||||
// Payload preprocessing
|
||||
const rawBody = payload.issue.body || '';
|
||||
const escapedBody = rawBody.replace(
|
||||
/<\/untrusted_context>/g,
|
||||
'\\</untrusted_context>',
|
||||
);
|
||||
const sanitizedBody = `<untrusted_context>\n${escapedBody}\n</untrusted_context>`;
|
||||
|
||||
const processedData = {
|
||||
issue_number: issueNumber,
|
||||
repository,
|
||||
sender: payload.sender?.login,
|
||||
body: sanitizedBody,
|
||||
title: payload.issue.title,
|
||||
};
|
||||
|
||||
const [owner, repo] = repository.split('/');
|
||||
const title = processedData.title || '';
|
||||
|
||||
try {
|
||||
const created = await issuesStore.createIssue(
|
||||
owner,
|
||||
repo,
|
||||
issueNumber,
|
||||
title,
|
||||
);
|
||||
|
||||
if (!created) {
|
||||
// If the Firestore document already exists, check its status.
|
||||
// If it is 'UNTRIAGED', we continue to publish to Pub/Sub
|
||||
// to recover from previous publish failures.
|
||||
const issueRef = issuesStore.getIssueRef(owner, repo, issueNumber);
|
||||
const snapshot = await issueRef.get();
|
||||
if (snapshot.get('status') !== 'UNTRIAGED') {
|
||||
return res.status(200).json({
|
||||
status: 'ignored',
|
||||
reason: `issue already exists: ${repository}#${issueNumber}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Publish to Pub/Sub
|
||||
const dataBuffer = Buffer.from(JSON.stringify(processedData));
|
||||
const messageId = await topic.publishMessage({ data: dataBuffer });
|
||||
|
||||
return res.status(202).json({ status: 'accepted', message_id: messageId });
|
||||
} catch (error) {
|
||||
console.error('Error processing webhook:', error);
|
||||
const message = error instanceof Error ? error.message : 'Unknown error';
|
||||
return res.status(500).json({ status: 'error', message });
|
||||
}
|
||||
});
|
||||
|
||||
// Global Express error handler for middleware failures (e.g., HTTP 413)
|
||||
app.use(
|
||||
(
|
||||
err: unknown,
|
||||
req: express.Request,
|
||||
res: express.Response,
|
||||
next: express.NextFunction,
|
||||
) => {
|
||||
if (
|
||||
err &&
|
||||
typeof err === 'object' &&
|
||||
'status' in err &&
|
||||
err.status === 413
|
||||
) {
|
||||
console.error('Payload too large. Limit is 1mb.');
|
||||
return res
|
||||
.status(413)
|
||||
.json({ status: 'error', message: 'Payload too large' });
|
||||
}
|
||||
next(err);
|
||||
},
|
||||
);
|
||||
|
||||
export { app };
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { verifyGithubSignature } from './github.js';
|
||||
import * as crypto from 'node:crypto';
|
||||
|
||||
describe('verifyGithubSignature', () => {
|
||||
const secret = 'my-secret';
|
||||
const payload = '{"test":true}';
|
||||
|
||||
it('should return true for a valid signature', () => {
|
||||
const hmac = crypto.createHmac('sha256', secret);
|
||||
hmac.update(payload);
|
||||
const validSignature = 'sha256=' + hmac.digest('hex');
|
||||
|
||||
const result = verifyGithubSignature(payload, validSignature, secret);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false if signatureHeader is missing', () => {
|
||||
const result = verifyGithubSignature(payload, undefined, secret);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false for an invalid signature', () => {
|
||||
const result = verifyGithubSignature(
|
||||
payload,
|
||||
'sha256=invalid-signature',
|
||||
secret,
|
||||
);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
|
||||
/**
|
||||
* Subset of the GitHub Webhook Payload for issues events.
|
||||
* @see https://docs.github.com/en/webhooks/webhook-events-and-payloads#issues
|
||||
*/
|
||||
export interface GitHubWebhookPayload {
|
||||
action: string;
|
||||
issue: {
|
||||
body?: string | null; // Can be null if description is empty
|
||||
number: number;
|
||||
title?: string;
|
||||
};
|
||||
repository: {
|
||||
/** Expected format: "owner/repo" (e.g. "google-gemini/gemini-cli") */
|
||||
full_name: string;
|
||||
};
|
||||
sender?: {
|
||||
login?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/** Regular expression matching standard GitHub repository format "owner/repo" */
|
||||
const GITHUB_REPO_REGEX = /^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/;
|
||||
|
||||
const GITHUB_SIGNATURE_HEADER_LENGTH = 71; // 'sha256=' (7) + 64 hex chars
|
||||
|
||||
/**
|
||||
* Verify that the payload was sent from GitHub using HMAC SHA256.
|
||||
*
|
||||
* @param payloadBody - The raw body of the request (Buffer or string).
|
||||
* @param signatureHeader - The value of the X-Hub-Signature-256 header.
|
||||
* @param secret - The GitHub Webhook secret.
|
||||
* @returns True if the signature is valid, false otherwise.
|
||||
* @see https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries
|
||||
*/
|
||||
export function verifyGithubSignature(
|
||||
payloadBody: Buffer | string,
|
||||
signatureHeader: string | undefined,
|
||||
secret: string,
|
||||
): boolean {
|
||||
if (
|
||||
!signatureHeader ||
|
||||
signatureHeader.length !== GITHUB_SIGNATURE_HEADER_LENGTH
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!Buffer.isBuffer(payloadBody) && typeof payloadBody !== 'string') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const hmac = crypto.createHmac('sha256', secret);
|
||||
hmac.update(payloadBody);
|
||||
const expectedSignature = 'sha256=' + hmac.digest('hex');
|
||||
|
||||
try {
|
||||
return crypto.timingSafeEqual(
|
||||
Buffer.from(expectedSignature),
|
||||
Buffer.from(signatureHeader),
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('Error verifying GitHub signature:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard to verify that an unknown object conforms to the GitHubWebhookPayload structure.
|
||||
*
|
||||
* @param obj - The object to validate.
|
||||
* @returns True if the object matches the schema, false otherwise.
|
||||
*/
|
||||
export function isGitHubWebhookPayload(
|
||||
obj: unknown,
|
||||
): obj is GitHubWebhookPayload {
|
||||
if (typeof obj !== 'object' || obj === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const o = obj as GitHubWebhookPayload;
|
||||
|
||||
// 1. Validate 'action'
|
||||
if (typeof o.action !== 'string') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 2. Validate 'issue'
|
||||
if (typeof o.issue !== 'object' || o.issue === null) {
|
||||
return false;
|
||||
}
|
||||
if (typeof o.issue.number !== 'number') {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
o.issue.body !== undefined &&
|
||||
o.issue.body !== null &&
|
||||
typeof o.issue.body !== 'string'
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (o.issue.title !== undefined && typeof o.issue.title !== 'string') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 3. Validate 'repository'
|
||||
if (typeof o.repository !== 'object' || o.repository === null) {
|
||||
return false;
|
||||
}
|
||||
if (typeof o.repository.full_name !== 'string') {
|
||||
return false;
|
||||
}
|
||||
if (!GITHUB_REPO_REGEX.test(o.repository.full_name)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 4. Validate 'sender' (optional)
|
||||
if (o.sender !== undefined) {
|
||||
if (typeof o.sender !== 'object' || o.sender === null) {
|
||||
return false;
|
||||
}
|
||||
if (o.sender.login !== undefined && typeof o.sender.login !== 'string') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import type { Mock } from 'vitest';
|
||||
import { IssuesStore } from './issuesStore.js';
|
||||
import type { Firestore, Transaction } from '@google-cloud/firestore';
|
||||
|
||||
describe('IssuesStore', () => {
|
||||
let mockTransaction: {
|
||||
get: Mock;
|
||||
set: Mock;
|
||||
};
|
||||
let mockDb: Firestore;
|
||||
let store: IssuesStore;
|
||||
|
||||
beforeEach(() => {
|
||||
// Assign mock read/write methods for transaction
|
||||
mockTransaction = {
|
||||
get: vi.fn(),
|
||||
set: vi.fn(),
|
||||
};
|
||||
|
||||
// Mock Firestore client
|
||||
mockDb = {
|
||||
collection: vi.fn().mockReturnThis(),
|
||||
doc: vi.fn().mockReturnValue({}),
|
||||
runTransaction: vi
|
||||
.fn()
|
||||
.mockImplementation((callback: (tx: Transaction) => Promise<unknown>) =>
|
||||
callback(mockTransaction as unknown as Transaction),
|
||||
),
|
||||
} as unknown as Firestore;
|
||||
|
||||
store = new IssuesStore(mockDb, 'issues-collection');
|
||||
});
|
||||
|
||||
it('should initialize a new issue if it does not exist', async () => {
|
||||
// The transaction should mock that the document does not exist
|
||||
mockTransaction.get.mockResolvedValue({ exists: false });
|
||||
|
||||
const result = await store.createIssue(
|
||||
'google',
|
||||
'gemini-cli',
|
||||
123,
|
||||
'Test Title',
|
||||
);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockTransaction.get).toHaveBeenCalled();
|
||||
expect(mockTransaction.set).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
status: 'UNTRIAGED',
|
||||
github_metadata: expect.objectContaining({
|
||||
owner: 'google',
|
||||
repo: 'gemini-cli',
|
||||
issue_number: 123,
|
||||
title: 'Test Title',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should return false and skip creation if the issue already exists', async () => {
|
||||
// The transaction should mock that the document already exists
|
||||
mockTransaction.get.mockResolvedValue({ exists: true });
|
||||
|
||||
const result = await store.createIssue(
|
||||
'google',
|
||||
'gemini-cli',
|
||||
123,
|
||||
'Test Title',
|
||||
);
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(mockTransaction.get).toHaveBeenCalled();
|
||||
expect(mockTransaction.set).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { FieldValue } from '@google-cloud/firestore';
|
||||
import type {
|
||||
Firestore,
|
||||
DocumentReference,
|
||||
Transaction,
|
||||
Timestamp,
|
||||
} from '@google-cloud/firestore';
|
||||
|
||||
export type IssueStatus =
|
||||
| 'UNTRIAGED'
|
||||
| 'TRIAGING'
|
||||
| 'NEEDS_INFO'
|
||||
| 'TRIAGED'
|
||||
| 'NEEDS_HUMAN'
|
||||
| 'LOW_QUALITY';
|
||||
|
||||
export interface IssueDocument {
|
||||
status: IssueStatus;
|
||||
triage_attempts: number;
|
||||
// The ingestion layer does not enforce the schema of workable_spec
|
||||
workable_spec: Record<string, unknown>;
|
||||
lock: {
|
||||
holder: string | null;
|
||||
expires_at: Timestamp | FieldValue | null;
|
||||
};
|
||||
created_at: Timestamp | FieldValue;
|
||||
updated_at: Timestamp | FieldValue;
|
||||
github_metadata: {
|
||||
owner: string;
|
||||
repo: string;
|
||||
issue_number: number;
|
||||
title: string;
|
||||
};
|
||||
}
|
||||
|
||||
export class IssuesStore {
|
||||
private readonly db: Firestore;
|
||||
private readonly collectionName: string;
|
||||
|
||||
constructor(db: Firestore, collectionName: string) {
|
||||
this.db = db;
|
||||
this.collectionName = collectionName;
|
||||
}
|
||||
|
||||
// Generates the standardized Firestore document reference for an issue
|
||||
getIssueRef(
|
||||
owner: string,
|
||||
repo: string,
|
||||
issueNumber: number,
|
||||
): DocumentReference {
|
||||
const docId = `github_${owner}_${repo}_${issueNumber}`;
|
||||
return this.db.collection(this.collectionName).doc(docId);
|
||||
}
|
||||
|
||||
// Initializes a new issue document in a transaction
|
||||
async createIssue(
|
||||
owner: string,
|
||||
repo: string,
|
||||
issueNumber: number,
|
||||
title: string,
|
||||
): Promise<boolean> {
|
||||
const docRef = this.getIssueRef(owner, repo, issueNumber);
|
||||
|
||||
try {
|
||||
return await this.db.runTransaction(async (transaction: Transaction) => {
|
||||
const snapshot = await transaction.get(docRef);
|
||||
|
||||
if (!snapshot.exists) {
|
||||
const newIssue: IssueDocument = {
|
||||
status: 'UNTRIAGED',
|
||||
triage_attempts: 0,
|
||||
workable_spec: {},
|
||||
lock: {
|
||||
holder: null,
|
||||
expires_at: null,
|
||||
},
|
||||
created_at: FieldValue.serverTimestamp(),
|
||||
updated_at: FieldValue.serverTimestamp(),
|
||||
github_metadata: {
|
||||
owner,
|
||||
repo,
|
||||
issue_number: issueNumber,
|
||||
title,
|
||||
},
|
||||
};
|
||||
|
||||
transaction.set(docRef, newIssue);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'Firestore transaction failed for issue:',
|
||||
`${owner}/${repo}#${issueNumber}`,
|
||||
error,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "ingestion-service",
|
||||
"version": "1.0.0",
|
||||
"description": "Ingestion service for triage worker",
|
||||
"main": "server.ts",
|
||||
"scripts": {
|
||||
"dev": "tsx watch server.ts",
|
||||
"build": "tsc",
|
||||
"start": "node dist/server.js",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google-cloud/firestore": "^7.7.0",
|
||||
"@google-cloud/pubsub": "^4.4.0",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.19.2",
|
||||
"express-rate-limit": "^7.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/express": "^4.17.21",
|
||||
"@types/node": "^20.12.12",
|
||||
"@types/supertest": "^6.0.3",
|
||||
"supertest": "^7.1.4",
|
||||
"tsx": "^4.9.3",
|
||||
"typescript": "^5.4.5",
|
||||
"vitest": "^1.6.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { app } from './app.js';
|
||||
|
||||
const port = parseInt(process.env.PORT || '8080', 10);
|
||||
app.listen(port, '0.0.0.0', () => {
|
||||
console.log(`Server listening on port ${port}`);
|
||||
});
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
Reference in New Issue
Block a user