Initial commit: Pulse RSS feed aggregator with fetcher module and all interfaces
This commit is contained in:
commit
84cafa9ecb
8
.gitignore
vendored
Normal file
8
.gitignore
vendored
Normal file
@ -0,0 +1,8 @@
|
||||
node_modules/
|
||||
.opencode/
|
||||
*.log
|
||||
.DS_Store
|
||||
dist/
|
||||
build/
|
||||
.env
|
||||
.env.local
|
||||
205
AGENTS.md
Normal file
205
AGENTS.md
Normal file
@ -0,0 +1,205 @@
|
||||
# Pulse — Agent Guide
|
||||
|
||||
RSS feed aggregator hosted at pulse.eazz.io.
|
||||
Built with a strict module-per-agent architecture. Each agent owns one module and communicates with others only through documented interfaces.
|
||||
|
||||
---
|
||||
|
||||
## Golden Rules
|
||||
|
||||
1. **Ask before you build.** If anything is unclear, ask. Do not assume. Do not start coding until you fully understand the task.
|
||||
2. **Plan before you execute.** Always produce a written plan first. Wait for approval before writing any code.
|
||||
3. **Stay in your module.** Do not read or write files outside your module folder unless you are the Orchestrator.
|
||||
4. **Respect the interfaces.** Never call internal functions of another module. Only use what is exported in `interfaces/`.
|
||||
5. **Small commits, clear messages.** One logical change per commit.
|
||||
6. **No silent assumptions.** If a requirement is ambiguous, surface it explicitly before proceeding.
|
||||
|
||||
---
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
pulse/
|
||||
├── AGENTS.md # This file
|
||||
├── interfaces/ # Shared contracts — source of truth for all modules
|
||||
│ ├── feed.types.ts # Core data types
|
||||
│ ├── fetcher.interface.ts # Fetcher module contract
|
||||
│ ├── parser.interface.ts # Parser module contract
|
||||
│ ├── storage.interface.ts # Storage module contract
|
||||
│ ├── dedup.interface.ts # Deduplication module contract
|
||||
│ └── formatter.interface.ts # Formatter module contract
|
||||
├── modules/
|
||||
│ ├── fetcher/ # HTTP fetching of RSS/Atom feeds
|
||||
│ ├── parser/ # XML parsing into FeedItem structs
|
||||
│ ├── dedup/ # Deduplication of feed items
|
||||
│ ├── storage/ # Persistence (SQLite)
|
||||
│ └── formatter/ # Output rendering (terminal / HTML)
|
||||
├── orchestrator/ # Coordinates modules, owns no business logic
|
||||
└── opencode.json # OpenCode agent config
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Core Data Types
|
||||
|
||||
These are defined in `interfaces/feed.types.ts` and are the lingua franca between all modules.
|
||||
|
||||
```typescript
|
||||
export interface FeedItem {
|
||||
id: string // Deterministic hash of url + publishedAt
|
||||
source: string // Feed origin URL
|
||||
title: string
|
||||
url: string
|
||||
publishedAt: Date
|
||||
content?: string // Optional full content
|
||||
summary?: string // Optional short summary
|
||||
}
|
||||
|
||||
export interface FetchError {
|
||||
source: string
|
||||
reason: string
|
||||
code: "NETWORK" | "TIMEOUT" | "PARSE" | "UNKNOWN"
|
||||
}
|
||||
|
||||
export interface FetchResult {
|
||||
items: FeedItem[]
|
||||
errors: FetchError[]
|
||||
fetchedAt: Date
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Module Interfaces
|
||||
|
||||
### Fetcher
|
||||
```typescript
|
||||
// interfaces/fetcher.interface.ts
|
||||
export interface IFetcher {
|
||||
fetch(feedUrl: string): Promise<FetchResult>
|
||||
fetchMany(feedUrls: string[]): Promise<FetchResult>
|
||||
}
|
||||
```
|
||||
|
||||
### Parser
|
||||
```typescript
|
||||
// interfaces/parser.interface.ts
|
||||
export interface IParser {
|
||||
parse(rawXml: string, source: string): Promise<FeedItem[]>
|
||||
supports(contentType: string): boolean // rss, atom, json feed
|
||||
}
|
||||
```
|
||||
|
||||
### Deduplication
|
||||
```typescript
|
||||
// interfaces/dedup.interface.ts
|
||||
export interface IDedup {
|
||||
filter(items: FeedItem[]): Promise<FeedItem[]> // returns only unseen items
|
||||
markSeen(items: FeedItem[]): Promise<void>
|
||||
}
|
||||
```
|
||||
|
||||
### Storage
|
||||
```typescript
|
||||
// interfaces/storage.interface.ts
|
||||
export interface IStorage {
|
||||
save(items: FeedItem[]): Promise<void>
|
||||
getRecent(limit: number): Promise<FeedItem[]>
|
||||
getBySource(source: string, limit: number): Promise<FeedItem[]>
|
||||
search(query: string): Promise<FeedItem[]>
|
||||
}
|
||||
```
|
||||
|
||||
### Formatter
|
||||
```typescript
|
||||
// interfaces/formatter.interface.ts
|
||||
export type OutputFormat = "terminal" | "html" | "json"
|
||||
|
||||
export interface IFormatter {
|
||||
format(items: FeedItem[], format: OutputFormat): Promise<string>
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Agent Responsibilities
|
||||
|
||||
| Agent | Owns | Can read interfaces | Cannot touch |
|
||||
|---|---|---|---|
|
||||
| `orchestrator` | `orchestrator/` | All | `modules/` internals |
|
||||
| `fetcher-agent` | `modules/fetcher/` | `feed.types`, `fetcher.interface` | All other modules |
|
||||
| `parser-agent` | `modules/parser/` | `feed.types`, `parser.interface` | All other modules |
|
||||
| `dedup-agent` | `modules/dedup/` | `feed.types`, `dedup.interface` | All other modules |
|
||||
| `storage-agent` | `modules/storage/` | `feed.types`, `storage.interface` | All other modules |
|
||||
| `formatter-agent` | `modules/formatter/` | `feed.types`, `formatter.interface` | All other modules |
|
||||
|
||||
---
|
||||
|
||||
## Workflow Every Agent Must Follow
|
||||
|
||||
### Step 1 — Clarify
|
||||
Before doing anything, re-read your task. If any of these are unclear, **ask**:
|
||||
- What exact input will I receive?
|
||||
- What exact output is expected?
|
||||
- Are there edge cases not mentioned?
|
||||
- Does this touch an interface that other modules depend on?
|
||||
|
||||
Do not proceed until you have answers.
|
||||
|
||||
### Step 2 — Plan
|
||||
Write a short implementation plan in plain text:
|
||||
- What files will you create or modify?
|
||||
- What are the key decisions and why?
|
||||
- What could go wrong?
|
||||
|
||||
Post the plan and **wait for approval** before writing any code.
|
||||
|
||||
### Step 3 — Implement
|
||||
Only after plan approval:
|
||||
- Stay inside your module folder
|
||||
- Implement against the interface, not against other module internals
|
||||
- Write tests alongside the code
|
||||
|
||||
### Step 4 — Verify
|
||||
- Run tests for your module only
|
||||
- Confirm the exported interface still matches `interfaces/`
|
||||
- Do not break existing interface contracts without flagging it first
|
||||
|
||||
---
|
||||
|
||||
## Interface Change Protocol
|
||||
|
||||
Interfaces in `interfaces/` are shared contracts. Changing them affects all modules.
|
||||
|
||||
If you need to change an interface:
|
||||
1. Stop. Do not change it unilaterally.
|
||||
2. Post a proposed change with reasoning to the orchestrator.
|
||||
3. Wait for explicit approval.
|
||||
4. Only then update the interface file AND all affected modules in the same commit.
|
||||
|
||||
---
|
||||
|
||||
## Tech Stack
|
||||
|
||||
- Runtime: Node.js (TypeScript)
|
||||
- Storage: SQLite via `better-sqlite3`
|
||||
- HTTP: `undici`
|
||||
- XML parsing: `fast-xml-parser`
|
||||
- Testing: `vitest`
|
||||
- Linting: `eslint` + `prettier`
|
||||
- Web: React + Tailwindcss + shadcn/ui
|
||||
- Server: Koa
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Questions Agents Should Ask Themselves
|
||||
|
||||
Before starting any task:
|
||||
- [ ] Do I fully understand what "done" looks like?
|
||||
- [ ] Have I read the relevant interface file?
|
||||
- [ ] Is my plan written and approved?
|
||||
- [ ] Am I staying inside my module?
|
||||
- [ ] Will my changes break any existing interface contract?
|
||||
|
||||
If any answer is "no" or "unsure" — ask before proceeding.
|
||||
19
interfaces/dedup.interface.ts
Normal file
19
interfaces/dedup.interface.ts
Normal file
@ -0,0 +1,19 @@
|
||||
import type { FeedItem } from './feed.types.js';
|
||||
|
||||
export interface DedupError {
|
||||
code: 'CACHE_ERROR' | 'UNKNOWN';
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface IDedup {
|
||||
/**
|
||||
* Returns only items that have not been seen before.
|
||||
* Does NOT mark them as seen - call markSeen() separately.
|
||||
*/
|
||||
filter(items: FeedItem[]): Promise<FeedItem[]>;
|
||||
|
||||
/**
|
||||
* Marks items as seen for future deduplication.
|
||||
*/
|
||||
markSeen(items: FeedItem[]): Promise<void>;
|
||||
}
|
||||
33
interfaces/feed.types.ts
Normal file
33
interfaces/feed.types.ts
Normal file
@ -0,0 +1,33 @@
|
||||
export interface FeedItem {
|
||||
id: string;
|
||||
source: string;
|
||||
title: string;
|
||||
url: string;
|
||||
publishedAt: Date;
|
||||
content?: string;
|
||||
summary?: string;
|
||||
}
|
||||
|
||||
export interface FetchInput {
|
||||
url: string;
|
||||
expectedFormat: "rss" | "atom";
|
||||
}
|
||||
|
||||
export interface FetchError {
|
||||
source: string;
|
||||
reason: string;
|
||||
code: "NETWORK" | "TIMEOUT" | "PARSE" | "UNKNOWN";
|
||||
}
|
||||
|
||||
export interface FeedResponse {
|
||||
source: string;
|
||||
body: string;
|
||||
contentType: string;
|
||||
statusCode: number;
|
||||
}
|
||||
|
||||
export interface FetchResult {
|
||||
responses: FeedResponse[];
|
||||
errors: FetchError[];
|
||||
fetchedAt: Date;
|
||||
}
|
||||
6
interfaces/fetcher.interface.ts
Normal file
6
interfaces/fetcher.interface.ts
Normal file
@ -0,0 +1,6 @@
|
||||
import type { FetchInput, FetchResult } from './feed.types.js';
|
||||
|
||||
export interface IFetcher {
|
||||
fetch(input: FetchInput): Promise<FetchResult>;
|
||||
fetchMany(inputs: FetchInput[]): Promise<FetchResult>;
|
||||
}
|
||||
16
interfaces/formatter.interface.ts
Normal file
16
interfaces/formatter.interface.ts
Normal file
@ -0,0 +1,16 @@
|
||||
import type { FeedItem } from './feed.types.js';
|
||||
|
||||
export type OutputFormat = 'terminal' | 'html' | 'json';
|
||||
|
||||
export interface FormatterError {
|
||||
code: 'SERIALIZE_ERROR' | 'UNKNOWN';
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface IFormatter {
|
||||
/**
|
||||
* Format items to the specified output format.
|
||||
* Returns string for terminal/html/json output.
|
||||
*/
|
||||
format(items: FeedItem[], format: OutputFormat): Promise<string>;
|
||||
}
|
||||
6
interfaces/parser.interface.ts
Normal file
6
interfaces/parser.interface.ts
Normal file
@ -0,0 +1,6 @@
|
||||
import type { FeedItem } from './feed.types.js';
|
||||
|
||||
export interface IParser {
|
||||
parse(rawXml: string, source: string): Promise<FeedItem[]>;
|
||||
supports(contentType: string): boolean;
|
||||
}
|
||||
28
interfaces/storage.interface.ts
Normal file
28
interfaces/storage.interface.ts
Normal file
@ -0,0 +1,28 @@
|
||||
import type { FeedItem } from './feed.types.js';
|
||||
|
||||
export interface StorageError {
|
||||
code: 'DB_ERROR' | 'CONSTRAINT_ERROR' | 'UNKNOWN';
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface IStorage {
|
||||
/**
|
||||
* Persist items to storage. Should handle duplicates gracefully (upsert).
|
||||
*/
|
||||
save(items: FeedItem[]): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get most recent items across all sources.
|
||||
*/
|
||||
getRecent(limit: number): Promise<FeedItem[]>;
|
||||
|
||||
/**
|
||||
* Get recent items from a specific source feed.
|
||||
*/
|
||||
getBySource(source: string, limit: number): Promise<FeedItem[]>;
|
||||
|
||||
/**
|
||||
* Search items by title/content keywords.
|
||||
*/
|
||||
search(query: string): Promise<FeedItem[]>;
|
||||
}
|
||||
211
modules/fetcher/fetcher.test.ts
Normal file
211
modules/fetcher/fetcher.test.ts
Normal file
@ -0,0 +1,211 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MockAgent } from 'undici';
|
||||
import { HttpFetcher } from './fetcher.js';
|
||||
|
||||
describe('HttpFetcher', () => {
|
||||
let mockAgent: MockAgent;
|
||||
let fetcher: HttpFetcher;
|
||||
|
||||
beforeEach(() => {
|
||||
mockAgent = new MockAgent();
|
||||
mockAgent.disableNetConnect();
|
||||
fetcher = new HttpFetcher({ timeout: 5000, dispatcher: mockAgent });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await mockAgent.close();
|
||||
});
|
||||
|
||||
it('returns response for valid RSS feed', async () => {
|
||||
const mockPool = mockAgent.get('https://example.com');
|
||||
mockPool
|
||||
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||
.reply(200, '<?xml version="1.0"?><rss></rss>', {
|
||||
headers: { 'content-type': 'application/rss+xml' },
|
||||
});
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://example.com/feed.xml',
|
||||
expectedFormat: 'rss',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(1);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
expect(result.responses[0].body).toBe('<?xml version="1.0"?><rss></rss>');
|
||||
expect(result.responses[0].contentType).toBe('application/rss+xml');
|
||||
expect(result.responses[0].statusCode).toBe(200);
|
||||
});
|
||||
|
||||
it('returns response for valid Atom feed', async () => {
|
||||
const mockPool = mockAgent.get('https://example.com');
|
||||
mockPool
|
||||
.intercept({ path: '/atom.xml', method: 'GET' })
|
||||
.reply(200, '<?xml version="1.0"?><feed></feed>', {
|
||||
headers: { 'content-type': 'application/atom+xml' },
|
||||
});
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://example.com/atom.xml',
|
||||
expectedFormat: 'atom',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(1);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
expect(result.responses[0].body).toBe('<?xml version="1.0"?><feed></feed>');
|
||||
});
|
||||
|
||||
it('returns PARSE error for wrong content type', async () => {
|
||||
const mockPool = mockAgent.get('https://example.com');
|
||||
mockPool
|
||||
.intercept({ path: '/feed.json', method: 'GET' })
|
||||
.reply(200, '{"foo":"bar"}', {
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://example.com/feed.json',
|
||||
expectedFormat: 'rss',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(0);
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.errors[0].code).toBe('PARSE');
|
||||
expect(result.errors[0].reason).toContain('Unexpected content type');
|
||||
});
|
||||
|
||||
it('returns PARSE error when body does not look like XML', async () => {
|
||||
const mockPool = mockAgent.get('https://example.com');
|
||||
mockPool
|
||||
.intercept({ path: '/plain.txt', method: 'GET' })
|
||||
.reply(200, 'not xml at all', {
|
||||
headers: { 'content-type': 'text/plain' },
|
||||
});
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://example.com/plain.txt',
|
||||
expectedFormat: 'rss',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(0);
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.errors[0].code).toBe('PARSE');
|
||||
});
|
||||
|
||||
it('returns UNKNOWN error for non-2xx status', async () => {
|
||||
const mockPool = mockAgent.get('https://example.com');
|
||||
mockPool
|
||||
.intercept({ path: '/notfound.xml', method: 'GET' })
|
||||
.reply(404, 'Not Found', {
|
||||
headers: { 'content-type': 'text/plain' },
|
||||
});
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://example.com/notfound.xml',
|
||||
expectedFormat: 'rss',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(0);
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.errors[0].code).toBe('UNKNOWN');
|
||||
expect(result.errors[0].reason).toContain('404');
|
||||
});
|
||||
|
||||
it('returns NETWORK error for connection refused', async () => {
|
||||
const mockPool = mockAgent.get('https://offline.com');
|
||||
const networkError = new Error('connect ECONNREFUSED 127.0.0.1:443');
|
||||
mockPool.intercept({ path: '/feed.xml', method: 'GET' }).replyWithError(networkError);
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://offline.com/feed.xml',
|
||||
expectedFormat: 'rss',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(0);
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.errors[0].code).toBe('NETWORK');
|
||||
});
|
||||
|
||||
it('returns TIMEOUT error for abort error', async () => {
|
||||
const mockPool = mockAgent.get('https://slow.com');
|
||||
const abortError = new Error('The operation was aborted');
|
||||
abortError.name = 'AbortError';
|
||||
mockPool.intercept({ path: '/feed.xml', method: 'GET' }).replyWithError(abortError);
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://slow.com/feed.xml',
|
||||
expectedFormat: 'rss',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(0);
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.errors[0].code).toBe('TIMEOUT');
|
||||
});
|
||||
|
||||
it('uses body sniff fallback when content-type is missing', async () => {
|
||||
const mockPool = mockAgent.get('https://example.com');
|
||||
mockPool
|
||||
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||
.reply(200, '<?xml version="1.0"?><rss></rss>', {
|
||||
headers: {},
|
||||
});
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://example.com/feed.xml',
|
||||
expectedFormat: 'rss',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(1);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
expect(result.responses[0].contentType).toBe('application/xml');
|
||||
});
|
||||
|
||||
it('accepts generic application/xml for RSS', async () => {
|
||||
const mockPool = mockAgent.get('https://example.com');
|
||||
mockPool
|
||||
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||
.reply(200, '<?xml version="1.0"?><rss></rss>', {
|
||||
headers: { 'content-type': 'application/xml' },
|
||||
});
|
||||
|
||||
const result = await fetcher.fetch({
|
||||
url: 'https://example.com/feed.xml',
|
||||
expectedFormat: 'rss',
|
||||
});
|
||||
|
||||
expect(result.responses).toHaveLength(1);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('processes multiple feeds with mixed results via fetchMany', async () => {
|
||||
const okPool = mockAgent.get('https://ok.com');
|
||||
okPool
|
||||
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||
.reply(200, '<?xml version="1.0"?><rss></rss>', {
|
||||
headers: { 'content-type': 'application/rss+xml' },
|
||||
});
|
||||
|
||||
const failPool = mockAgent.get('https://fail.com');
|
||||
failPool
|
||||
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||
.reply(500, 'Internal Server Error', {
|
||||
headers: { 'content-type': 'text/plain' },
|
||||
});
|
||||
|
||||
const result = await fetcher.fetchMany([
|
||||
{ url: 'https://ok.com/feed.xml', expectedFormat: 'rss' },
|
||||
{ url: 'https://fail.com/feed.xml', expectedFormat: 'rss' },
|
||||
]);
|
||||
|
||||
expect(result.responses).toHaveLength(1);
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.responses[0].source).toBe('https://ok.com/feed.xml');
|
||||
expect(result.errors[0].source).toBe('https://fail.com/feed.xml');
|
||||
});
|
||||
|
||||
it('returns empty result for empty input array', async () => {
|
||||
const result = await fetcher.fetchMany([]);
|
||||
|
||||
expect(result.responses).toHaveLength(0);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
178
modules/fetcher/fetcher.ts
Normal file
178
modules/fetcher/fetcher.ts
Normal file
@ -0,0 +1,178 @@
|
||||
import { request, type Dispatcher } from 'undici';
|
||||
import type { FetchInput, FetchResult, FeedResponse, FetchError } from '../../interfaces/feed.types.js';
|
||||
import type { IFetcher } from '../../interfaces/fetcher.interface.js';
|
||||
|
||||
const VALID_RSS_CONTENT_TYPES = [
|
||||
'application/xml',
|
||||
'text/xml',
|
||||
'application/rss+xml',
|
||||
];
|
||||
|
||||
const VALID_ATOM_CONTENT_TYPES = [
|
||||
'application/xml',
|
||||
'text/xml',
|
||||
'application/atom+xml',
|
||||
];
|
||||
|
||||
function getValidTypes(expectedFormat: FetchInput['expectedFormat']): string[] {
|
||||
return expectedFormat === 'rss' ? VALID_RSS_CONTENT_TYPES : VALID_ATOM_CONTENT_TYPES;
|
||||
}
|
||||
|
||||
function normalizeContentType(contentType: string): string {
|
||||
return contentType.split(';')[0].trim().toLowerCase();
|
||||
}
|
||||
|
||||
function isValidFormat(expectedFormat: FetchInput['expectedFormat'], contentType: string): boolean {
|
||||
const normalized = normalizeContentType(contentType);
|
||||
const validTypes = getValidTypes(expectedFormat);
|
||||
return validTypes.includes(normalized);
|
||||
}
|
||||
|
||||
function bodyLooksLikeXml(body: string): boolean {
|
||||
const trimmed = body.trimStart().toLowerCase();
|
||||
return trimmed.startsWith('<?xml') || trimmed.startsWith('<rss') || trimmed.startsWith('<feed');
|
||||
}
|
||||
|
||||
function classifyError(error: unknown, source: string): FetchError {
|
||||
const err = error as Error & { code?: string };
|
||||
const message = err.message ?? String(error);
|
||||
|
||||
if (
|
||||
err.name === 'AbortError' ||
|
||||
err.code === 'UND_ERR_CONNECT_TIMEOUT' ||
|
||||
err.code === 'UND_ERR_REQUEST_TIMEOUT' ||
|
||||
err.code === 'UND_ERR_HEADERS_TIMEOUT'
|
||||
) {
|
||||
return { source, reason: message, code: 'TIMEOUT' };
|
||||
}
|
||||
|
||||
const networkCodes = ['ECONNREFUSED', 'ENOTFOUND', 'ETIMEDOUT', 'EAI_AGAIN', 'ECONNRESET', 'EPIPE', 'ENETUNREACH'];
|
||||
if (
|
||||
networkCodes.includes(err.code ?? '') ||
|
||||
networkCodes.some((code) => message.includes(code))
|
||||
) {
|
||||
return { source, reason: message, code: 'NETWORK' };
|
||||
}
|
||||
|
||||
return { source, reason: message, code: 'UNKNOWN' };
|
||||
}
|
||||
|
||||
export interface HttpFetcherOptions {
|
||||
timeout?: number;
|
||||
concurrency?: number;
|
||||
dispatcher?: Dispatcher;
|
||||
}
|
||||
|
||||
export class HttpFetcher implements IFetcher {
|
||||
private readonly timeout: number;
|
||||
private readonly concurrency: number;
|
||||
private readonly dispatcher?: Dispatcher;
|
||||
|
||||
constructor(options: HttpFetcherOptions = {}) {
|
||||
this.timeout = options.timeout ?? 10_000;
|
||||
this.concurrency = options.concurrency ?? 5;
|
||||
this.dispatcher = options.dispatcher;
|
||||
}
|
||||
|
||||
async fetch(input: FetchInput): Promise<FetchResult> {
|
||||
const result: FetchResult = { responses: [], errors: [], fetchedAt: new Date() };
|
||||
|
||||
try {
|
||||
const { statusCode, headers, body } = await request(input.url, {
|
||||
method: 'GET',
|
||||
headers: { 'User-Agent': 'Pulse-RSS-Fetcher/1.0' },
|
||||
signal: AbortSignal.timeout(this.timeout),
|
||||
dispatcher: this.dispatcher,
|
||||
});
|
||||
|
||||
const responseBody = await body.text();
|
||||
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
result.errors.push({
|
||||
source: input.url,
|
||||
reason: `HTTP ${statusCode}`,
|
||||
code: 'UNKNOWN',
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
const contentType = (headers['content-type'] as string | undefined) ?? '';
|
||||
|
||||
if (contentType) {
|
||||
if (!isValidFormat(input.expectedFormat, contentType)) {
|
||||
if (!bodyLooksLikeXml(responseBody)) {
|
||||
result.errors.push({
|
||||
source: input.url,
|
||||
reason: `Unexpected content type: ${contentType}`,
|
||||
code: 'PARSE',
|
||||
});
|
||||
return result;
|
||||
}
|
||||
}
|
||||
} else if (!bodyLooksLikeXml(responseBody)) {
|
||||
result.errors.push({
|
||||
source: input.url,
|
||||
reason: 'Response does not appear to be XML',
|
||||
code: 'PARSE',
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
result.responses.push({
|
||||
source: input.url,
|
||||
body: responseBody,
|
||||
contentType: contentType || 'application/xml',
|
||||
statusCode,
|
||||
});
|
||||
} catch (error) {
|
||||
result.errors.push(classifyError(error, input.url));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async fetchMany(inputs: FetchInput[]): Promise<FetchResult> {
|
||||
const result: FetchResult = { responses: [], errors: [], fetchedAt: new Date() };
|
||||
const limit = createConcurrencyLimit(this.concurrency);
|
||||
|
||||
const results = await Promise.all(inputs.map((input) => limit(() => this.fetch(input))));
|
||||
|
||||
for (const r of results) {
|
||||
result.responses.push(...r.responses);
|
||||
result.errors.push(...r.errors);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
function createConcurrencyLimit(concurrency: number) {
|
||||
const queue: (() => void)[] = [];
|
||||
let activeCount = 0;
|
||||
|
||||
return function <T>(fn: () => Promise<T>): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const run = async () => {
|
||||
activeCount++;
|
||||
try {
|
||||
const result = await fn();
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
} finally {
|
||||
activeCount--;
|
||||
if (queue.length > 0) {
|
||||
const next = queue.shift()!;
|
||||
next();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (activeCount < concurrency) {
|
||||
run();
|
||||
} else {
|
||||
queue.push(run);
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
1
modules/fetcher/index.ts
Normal file
1
modules/fetcher/index.ts
Normal file
@ -0,0 +1 @@
|
||||
export { HttpFetcher, type HttpFetcherOptions } from './fetcher.js';
|
||||
161
opencode.json
Normal file
161
opencode.json
Normal file
@ -0,0 +1,161 @@
|
||||
{
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"agent": {
|
||||
"build": {
|
||||
"mode": "primary",
|
||||
"model": "opencode/kimi-k2.5",
|
||||
"prompt": "{file:./.opencode/prompts/build.txt}",
|
||||
"permission": {
|
||||
"edit": "allow",
|
||||
"bash": {
|
||||
"*": "ask",
|
||||
"npm run *": "allow",
|
||||
"npx vitest *": "allow",
|
||||
"npx tsc *": "allow",
|
||||
"git status": "allow",
|
||||
"git diff *": "allow",
|
||||
"git log *": "allow",
|
||||
"git add *": "ask",
|
||||
"git commit *": "ask",
|
||||
"git push *": "ask"
|
||||
}
|
||||
}
|
||||
},
|
||||
"plan": {
|
||||
"mode": "primary",
|
||||
"model": "opencode/kimi-k2.5",
|
||||
"temperature": 0.1,
|
||||
"prompt": "{file:./.opencode/prompts/plan.txt}",
|
||||
"permission": {
|
||||
"edit": "deny",
|
||||
"bash": "deny"
|
||||
}
|
||||
},
|
||||
"orchestrator": {
|
||||
"description": "Coordinates work across all Pulse modules. Plans tasks, delegates to module agents, owns no business logic.",
|
||||
"mode": "subagent",
|
||||
"model": "opencode/kimi-k2.5",
|
||||
"temperature": 0.1,
|
||||
"prompt": "{file:./.opencode/prompts/orchestrator.txt}",
|
||||
"permission": {
|
||||
"edit": "deny",
|
||||
"bash": "deny",
|
||||
"task": {
|
||||
"*": "allow"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fetcher-agent": {
|
||||
"description": "Implements and maintains the fetcher module (modules/fetcher/). Handles HTTP fetching of RSS and Atom feeds.",
|
||||
"mode": "subagent",
|
||||
"model": "opencode/kimi-k2.5",
|
||||
"temperature": 0.2,
|
||||
"prompt": "{file:./.opencode/prompts/fetcher.txt}",
|
||||
"permission": {
|
||||
"edit": {
|
||||
"modules/fetcher/**": "allow",
|
||||
"interfaces/**": "ask",
|
||||
"**": "deny"
|
||||
},
|
||||
"bash": {
|
||||
"npx vitest run modules/fetcher*": "allow",
|
||||
"npx tsc --noEmit": "allow",
|
||||
"*": "deny"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parser-agent": {
|
||||
"description": "Implements and maintains the parser module (modules/parser/). Converts raw XML/Atom into FeedItem structs.",
|
||||
"mode": "subagent",
|
||||
"model": "opencode/kimi-k2.5",
|
||||
"temperature": 0.2,
|
||||
"prompt": "{file:./.opencode/prompts/parser.txt}",
|
||||
"permission": {
|
||||
"edit": {
|
||||
"modules/parser/**": "allow",
|
||||
"interfaces/**": "ask",
|
||||
"**": "deny"
|
||||
},
|
||||
"bash": {
|
||||
"npx vitest run modules/parser*": "allow",
|
||||
"npx tsc --noEmit": "allow",
|
||||
"*": "deny"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dedup-agent": {
|
||||
"description": "Implements and maintains the deduplication module (modules/dedup/). Filters already-seen feed items.",
|
||||
"mode": "subagent",
|
||||
"model": "opencode/kimi-k2.5",
|
||||
"temperature": 0.2,
|
||||
"prompt": "{file:./.opencode/prompts/dedup.txt}",
|
||||
"permission": {
|
||||
"edit": {
|
||||
"modules/dedup/**": "allow",
|
||||
"interfaces/**": "ask",
|
||||
"**": "deny"
|
||||
},
|
||||
"bash": {
|
||||
"npx vitest run modules/dedup*": "allow",
|
||||
"npx tsc --noEmit": "allow",
|
||||
"*": "deny"
|
||||
}
|
||||
}
|
||||
},
|
||||
"storage-agent": {
|
||||
"description": "Implements and maintains the storage module (modules/storage/). Persists FeedItems to SQLite.",
|
||||
"mode": "subagent",
|
||||
"model": "opencode/kimi-k2.5",
|
||||
"temperature": 0.2,
|
||||
"prompt": "{file:./.opencode/prompts/storage.txt}",
|
||||
"permission": {
|
||||
"edit": {
|
||||
"modules/storage/**": "allow",
|
||||
"interfaces/**": "ask",
|
||||
"**": "deny"
|
||||
},
|
||||
"bash": {
|
||||
"npx vitest run modules/storage*": "allow",
|
||||
"npx tsc --noEmit": "allow",
|
||||
"*": "deny"
|
||||
}
|
||||
}
|
||||
},
|
||||
"formatter-agent": {
|
||||
"description": "Implements and maintains the formatter module (modules/formatter/). Renders FeedItems to terminal, HTML, or JSON.",
|
||||
"mode": "subagent",
|
||||
"model": "opencode/kimi-k2.5",
|
||||
"temperature": 0.2,
|
||||
"prompt": "{file:./.opencode/prompts/formatter.txt}",
|
||||
"permission": {
|
||||
"edit": {
|
||||
"modules/formatter/**": "allow",
|
||||
"interfaces/**": "ask",
|
||||
"**": "deny"
|
||||
},
|
||||
"bash": {
|
||||
"npx vitest run modules/formatter*": "allow",
|
||||
"npx tsc --noEmit": "allow",
|
||||
"*": "deny"
|
||||
}
|
||||
}
|
||||
},
|
||||
"reviewer": {
|
||||
"description": "Reviews code for quality, correctness, and interface compliance. Read-only.",
|
||||
"mode": "subagent",
|
||||
"model": "opencode/claude-sonnet-4-6",
|
||||
"temperature": 0.1,
|
||||
"prompt": "{file:./.opencode/prompts/reviewer.txt}",
|
||||
"color": "accent",
|
||||
"permission": {
|
||||
"edit": "deny",
|
||||
"bash": {
|
||||
"npx tsc --noEmit": "allow",
|
||||
"npx vitest run": "allow",
|
||||
"git diff *": "allow",
|
||||
"*": "deny"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
1994
package-lock.json
generated
Normal file
1994
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
19
package.json
Normal file
19
package.json
Normal file
@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "pulse",
|
||||
"version": "0.1.0",
|
||||
"description": "RSS feed aggregator",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.7.0",
|
||||
"vitest": "^2.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"undici": "^6.21.0"
|
||||
}
|
||||
}
|
||||
20
tsconfig.json
Normal file
20
tsconfig.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
8
vitest.config.ts
Normal file
8
vitest.config.ts
Normal file
@ -0,0 +1,8 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
},
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user