Initial commit: Pulse RSS feed aggregator with fetcher module and all interfaces
This commit is contained in:
commit
84cafa9ecb
8
.gitignore
vendored
Normal file
8
.gitignore
vendored
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
node_modules/
|
||||||
|
.opencode/
|
||||||
|
*.log
|
||||||
|
.DS_Store
|
||||||
|
dist/
|
||||||
|
build/
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
205
AGENTS.md
Normal file
205
AGENTS.md
Normal file
@ -0,0 +1,205 @@
|
|||||||
|
# Pulse — Agent Guide
|
||||||
|
|
||||||
|
RSS feed aggregator hosted at pulse.eazz.io.
|
||||||
|
Built with a strict module-per-agent architecture. Each agent owns one module and communicates with others only through documented interfaces.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Golden Rules
|
||||||
|
|
||||||
|
1. **Ask before you build.** If anything is unclear, ask. Do not assume. Do not start coding until you fully understand the task.
|
||||||
|
2. **Plan before you execute.** Always produce a written plan first. Wait for approval before writing any code.
|
||||||
|
3. **Stay in your module.** Do not read or write files outside your module folder unless you are the Orchestrator.
|
||||||
|
4. **Respect the interfaces.** Never call internal functions of another module. Only use what is exported in `interfaces/`.
|
||||||
|
5. **Small commits, clear messages.** One logical change per commit.
|
||||||
|
6. **No silent assumptions.** If a requirement is ambiguous, surface it explicitly before proceeding.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Project Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
pulse/
|
||||||
|
├── AGENTS.md # This file
|
||||||
|
├── interfaces/ # Shared contracts — source of truth for all modules
|
||||||
|
│ ├── feed.types.ts # Core data types
|
||||||
|
│ ├── fetcher.interface.ts # Fetcher module contract
|
||||||
|
│ ├── parser.interface.ts # Parser module contract
|
||||||
|
│ ├── storage.interface.ts # Storage module contract
|
||||||
|
│ ├── dedup.interface.ts # Deduplication module contract
|
||||||
|
│ └── formatter.interface.ts # Formatter module contract
|
||||||
|
├── modules/
|
||||||
|
│ ├── fetcher/ # HTTP fetching of RSS/Atom feeds
|
||||||
|
│ ├── parser/ # XML parsing into FeedItem structs
|
||||||
|
│ ├── dedup/ # Deduplication of feed items
|
||||||
|
│ ├── storage/ # Persistence (SQLite)
|
||||||
|
│ └── formatter/ # Output rendering (terminal / HTML)
|
||||||
|
├── orchestrator/ # Coordinates modules, owns no business logic
|
||||||
|
└── opencode.json # OpenCode agent config
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Core Data Types
|
||||||
|
|
||||||
|
These are defined in `interfaces/feed.types.ts` and are the lingua franca between all modules.
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
export interface FeedItem {
|
||||||
|
id: string // Deterministic hash of url + publishedAt
|
||||||
|
source: string // Feed origin URL
|
||||||
|
title: string
|
||||||
|
url: string
|
||||||
|
publishedAt: Date
|
||||||
|
content?: string // Optional full content
|
||||||
|
summary?: string // Optional short summary
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FetchError {
|
||||||
|
source: string
|
||||||
|
reason: string
|
||||||
|
code: "NETWORK" | "TIMEOUT" | "PARSE" | "UNKNOWN"
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FetchResult {
|
||||||
|
items: FeedItem[]
|
||||||
|
errors: FetchError[]
|
||||||
|
fetchedAt: Date
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Module Interfaces
|
||||||
|
|
||||||
|
### Fetcher
|
||||||
|
```typescript
|
||||||
|
// interfaces/fetcher.interface.ts
|
||||||
|
export interface IFetcher {
|
||||||
|
fetch(feedUrl: string): Promise<FetchResult>
|
||||||
|
fetchMany(feedUrls: string[]): Promise<FetchResult>
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Parser
|
||||||
|
```typescript
|
||||||
|
// interfaces/parser.interface.ts
|
||||||
|
export interface IParser {
|
||||||
|
parse(rawXml: string, source: string): Promise<FeedItem[]>
|
||||||
|
supports(contentType: string): boolean // rss, atom, json feed
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Deduplication
|
||||||
|
```typescript
|
||||||
|
// interfaces/dedup.interface.ts
|
||||||
|
export interface IDedup {
|
||||||
|
filter(items: FeedItem[]): Promise<FeedItem[]> // returns only unseen items
|
||||||
|
markSeen(items: FeedItem[]): Promise<void>
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Storage
|
||||||
|
```typescript
|
||||||
|
// interfaces/storage.interface.ts
|
||||||
|
export interface IStorage {
|
||||||
|
save(items: FeedItem[]): Promise<void>
|
||||||
|
getRecent(limit: number): Promise<FeedItem[]>
|
||||||
|
getBySource(source: string, limit: number): Promise<FeedItem[]>
|
||||||
|
search(query: string): Promise<FeedItem[]>
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Formatter
|
||||||
|
```typescript
|
||||||
|
// interfaces/formatter.interface.ts
|
||||||
|
export type OutputFormat = "terminal" | "html" | "json"
|
||||||
|
|
||||||
|
export interface IFormatter {
|
||||||
|
format(items: FeedItem[], format: OutputFormat): Promise<string>
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Agent Responsibilities
|
||||||
|
|
||||||
|
| Agent | Owns | Can read interfaces | Cannot touch |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `orchestrator` | `orchestrator/` | All | `modules/` internals |
|
||||||
|
| `fetcher-agent` | `modules/fetcher/` | `feed.types`, `fetcher.interface` | All other modules |
|
||||||
|
| `parser-agent` | `modules/parser/` | `feed.types`, `parser.interface` | All other modules |
|
||||||
|
| `dedup-agent` | `modules/dedup/` | `feed.types`, `dedup.interface` | All other modules |
|
||||||
|
| `storage-agent` | `modules/storage/` | `feed.types`, `storage.interface` | All other modules |
|
||||||
|
| `formatter-agent` | `modules/formatter/` | `feed.types`, `formatter.interface` | All other modules |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Workflow Every Agent Must Follow
|
||||||
|
|
||||||
|
### Step 1 — Clarify
|
||||||
|
Before doing anything, re-read your task. If any of these are unclear, **ask**:
|
||||||
|
- What exact input will I receive?
|
||||||
|
- What exact output is expected?
|
||||||
|
- Are there edge cases not mentioned?
|
||||||
|
- Does this touch an interface that other modules depend on?
|
||||||
|
|
||||||
|
Do not proceed until you have answers.
|
||||||
|
|
||||||
|
### Step 2 — Plan
|
||||||
|
Write a short implementation plan in plain text:
|
||||||
|
- What files will you create or modify?
|
||||||
|
- What are the key decisions and why?
|
||||||
|
- What could go wrong?
|
||||||
|
|
||||||
|
Post the plan and **wait for approval** before writing any code.
|
||||||
|
|
||||||
|
### Step 3 — Implement
|
||||||
|
Only after plan approval:
|
||||||
|
- Stay inside your module folder
|
||||||
|
- Implement against the interface, not against other module internals
|
||||||
|
- Write tests alongside the code
|
||||||
|
|
||||||
|
### Step 4 — Verify
|
||||||
|
- Run tests for your module only
|
||||||
|
- Confirm the exported interface still matches `interfaces/`
|
||||||
|
- Do not break existing interface contracts without flagging it first
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Interface Change Protocol
|
||||||
|
|
||||||
|
Interfaces in `interfaces/` are shared contracts. Changing them affects all modules.
|
||||||
|
|
||||||
|
If you need to change an interface:
|
||||||
|
1. Stop. Do not change it unilaterally.
|
||||||
|
2. Post a proposed change with reasoning to the orchestrator.
|
||||||
|
3. Wait for explicit approval.
|
||||||
|
4. Only then update the interface file AND all affected modules in the same commit.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
|
||||||
|
- Runtime: Node.js (TypeScript)
|
||||||
|
- Storage: SQLite via `better-sqlite3`
|
||||||
|
- HTTP: `undici`
|
||||||
|
- XML parsing: `fast-xml-parser`
|
||||||
|
- Testing: `vitest`
|
||||||
|
- Linting: `eslint` + `prettier`
|
||||||
|
- Web: React + Tailwindcss + shadcn/ui
|
||||||
|
- Server: Koa
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Questions Agents Should Ask Themselves
|
||||||
|
|
||||||
|
Before starting any task:
|
||||||
|
- [ ] Do I fully understand what "done" looks like?
|
||||||
|
- [ ] Have I read the relevant interface file?
|
||||||
|
- [ ] Is my plan written and approved?
|
||||||
|
- [ ] Am I staying inside my module?
|
||||||
|
- [ ] Will my changes break any existing interface contract?
|
||||||
|
|
||||||
|
If any answer is "no" or "unsure" — ask before proceeding.
|
||||||
19
interfaces/dedup.interface.ts
Normal file
19
interfaces/dedup.interface.ts
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
import type { FeedItem } from './feed.types.js';
|
||||||
|
|
||||||
|
export interface DedupError {
|
||||||
|
code: 'CACHE_ERROR' | 'UNKNOWN';
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IDedup {
|
||||||
|
/**
|
||||||
|
* Returns only items that have not been seen before.
|
||||||
|
* Does NOT mark them as seen - call markSeen() separately.
|
||||||
|
*/
|
||||||
|
filter(items: FeedItem[]): Promise<FeedItem[]>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Marks items as seen for future deduplication.
|
||||||
|
*/
|
||||||
|
markSeen(items: FeedItem[]): Promise<void>;
|
||||||
|
}
|
||||||
33
interfaces/feed.types.ts
Normal file
33
interfaces/feed.types.ts
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
export interface FeedItem {
|
||||||
|
id: string;
|
||||||
|
source: string;
|
||||||
|
title: string;
|
||||||
|
url: string;
|
||||||
|
publishedAt: Date;
|
||||||
|
content?: string;
|
||||||
|
summary?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FetchInput {
|
||||||
|
url: string;
|
||||||
|
expectedFormat: "rss" | "atom";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FetchError {
|
||||||
|
source: string;
|
||||||
|
reason: string;
|
||||||
|
code: "NETWORK" | "TIMEOUT" | "PARSE" | "UNKNOWN";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FeedResponse {
|
||||||
|
source: string;
|
||||||
|
body: string;
|
||||||
|
contentType: string;
|
||||||
|
statusCode: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FetchResult {
|
||||||
|
responses: FeedResponse[];
|
||||||
|
errors: FetchError[];
|
||||||
|
fetchedAt: Date;
|
||||||
|
}
|
||||||
6
interfaces/fetcher.interface.ts
Normal file
6
interfaces/fetcher.interface.ts
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
import type { FetchInput, FetchResult } from './feed.types.js';
|
||||||
|
|
||||||
|
export interface IFetcher {
|
||||||
|
fetch(input: FetchInput): Promise<FetchResult>;
|
||||||
|
fetchMany(inputs: FetchInput[]): Promise<FetchResult>;
|
||||||
|
}
|
||||||
16
interfaces/formatter.interface.ts
Normal file
16
interfaces/formatter.interface.ts
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
import type { FeedItem } from './feed.types.js';
|
||||||
|
|
||||||
|
export type OutputFormat = 'terminal' | 'html' | 'json';
|
||||||
|
|
||||||
|
export interface FormatterError {
|
||||||
|
code: 'SERIALIZE_ERROR' | 'UNKNOWN';
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IFormatter {
|
||||||
|
/**
|
||||||
|
* Format items to the specified output format.
|
||||||
|
* Returns string for terminal/html/json output.
|
||||||
|
*/
|
||||||
|
format(items: FeedItem[], format: OutputFormat): Promise<string>;
|
||||||
|
}
|
||||||
6
interfaces/parser.interface.ts
Normal file
6
interfaces/parser.interface.ts
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
import type { FeedItem } from './feed.types.js';
|
||||||
|
|
||||||
|
export interface IParser {
|
||||||
|
parse(rawXml: string, source: string): Promise<FeedItem[]>;
|
||||||
|
supports(contentType: string): boolean;
|
||||||
|
}
|
||||||
28
interfaces/storage.interface.ts
Normal file
28
interfaces/storage.interface.ts
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
import type { FeedItem } from './feed.types.js';
|
||||||
|
|
||||||
|
export interface StorageError {
|
||||||
|
code: 'DB_ERROR' | 'CONSTRAINT_ERROR' | 'UNKNOWN';
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IStorage {
|
||||||
|
/**
|
||||||
|
* Persist items to storage. Should handle duplicates gracefully (upsert).
|
||||||
|
*/
|
||||||
|
save(items: FeedItem[]): Promise<void>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get most recent items across all sources.
|
||||||
|
*/
|
||||||
|
getRecent(limit: number): Promise<FeedItem[]>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get recent items from a specific source feed.
|
||||||
|
*/
|
||||||
|
getBySource(source: string, limit: number): Promise<FeedItem[]>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Search items by title/content keywords.
|
||||||
|
*/
|
||||||
|
search(query: string): Promise<FeedItem[]>;
|
||||||
|
}
|
||||||
211
modules/fetcher/fetcher.test.ts
Normal file
211
modules/fetcher/fetcher.test.ts
Normal file
@ -0,0 +1,211 @@
|
|||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||||
|
import { MockAgent } from 'undici';
|
||||||
|
import { HttpFetcher } from './fetcher.js';
|
||||||
|
|
||||||
|
describe('HttpFetcher', () => {
|
||||||
|
let mockAgent: MockAgent;
|
||||||
|
let fetcher: HttpFetcher;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
mockAgent = new MockAgent();
|
||||||
|
mockAgent.disableNetConnect();
|
||||||
|
fetcher = new HttpFetcher({ timeout: 5000, dispatcher: mockAgent });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await mockAgent.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns response for valid RSS feed', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://example.com');
|
||||||
|
mockPool
|
||||||
|
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||||
|
.reply(200, '<?xml version="1.0"?><rss></rss>', {
|
||||||
|
headers: { 'content-type': 'application/rss+xml' },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://example.com/feed.xml',
|
||||||
|
expectedFormat: 'rss',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(1);
|
||||||
|
expect(result.errors).toHaveLength(0);
|
||||||
|
expect(result.responses[0].body).toBe('<?xml version="1.0"?><rss></rss>');
|
||||||
|
expect(result.responses[0].contentType).toBe('application/rss+xml');
|
||||||
|
expect(result.responses[0].statusCode).toBe(200);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns response for valid Atom feed', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://example.com');
|
||||||
|
mockPool
|
||||||
|
.intercept({ path: '/atom.xml', method: 'GET' })
|
||||||
|
.reply(200, '<?xml version="1.0"?><feed></feed>', {
|
||||||
|
headers: { 'content-type': 'application/atom+xml' },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://example.com/atom.xml',
|
||||||
|
expectedFormat: 'atom',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(1);
|
||||||
|
expect(result.errors).toHaveLength(0);
|
||||||
|
expect(result.responses[0].body).toBe('<?xml version="1.0"?><feed></feed>');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns PARSE error for wrong content type', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://example.com');
|
||||||
|
mockPool
|
||||||
|
.intercept({ path: '/feed.json', method: 'GET' })
|
||||||
|
.reply(200, '{"foo":"bar"}', {
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://example.com/feed.json',
|
||||||
|
expectedFormat: 'rss',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(0);
|
||||||
|
expect(result.errors).toHaveLength(1);
|
||||||
|
expect(result.errors[0].code).toBe('PARSE');
|
||||||
|
expect(result.errors[0].reason).toContain('Unexpected content type');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns PARSE error when body does not look like XML', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://example.com');
|
||||||
|
mockPool
|
||||||
|
.intercept({ path: '/plain.txt', method: 'GET' })
|
||||||
|
.reply(200, 'not xml at all', {
|
||||||
|
headers: { 'content-type': 'text/plain' },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://example.com/plain.txt',
|
||||||
|
expectedFormat: 'rss',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(0);
|
||||||
|
expect(result.errors).toHaveLength(1);
|
||||||
|
expect(result.errors[0].code).toBe('PARSE');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns UNKNOWN error for non-2xx status', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://example.com');
|
||||||
|
mockPool
|
||||||
|
.intercept({ path: '/notfound.xml', method: 'GET' })
|
||||||
|
.reply(404, 'Not Found', {
|
||||||
|
headers: { 'content-type': 'text/plain' },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://example.com/notfound.xml',
|
||||||
|
expectedFormat: 'rss',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(0);
|
||||||
|
expect(result.errors).toHaveLength(1);
|
||||||
|
expect(result.errors[0].code).toBe('UNKNOWN');
|
||||||
|
expect(result.errors[0].reason).toContain('404');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns NETWORK error for connection refused', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://offline.com');
|
||||||
|
const networkError = new Error('connect ECONNREFUSED 127.0.0.1:443');
|
||||||
|
mockPool.intercept({ path: '/feed.xml', method: 'GET' }).replyWithError(networkError);
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://offline.com/feed.xml',
|
||||||
|
expectedFormat: 'rss',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(0);
|
||||||
|
expect(result.errors).toHaveLength(1);
|
||||||
|
expect(result.errors[0].code).toBe('NETWORK');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns TIMEOUT error for abort error', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://slow.com');
|
||||||
|
const abortError = new Error('The operation was aborted');
|
||||||
|
abortError.name = 'AbortError';
|
||||||
|
mockPool.intercept({ path: '/feed.xml', method: 'GET' }).replyWithError(abortError);
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://slow.com/feed.xml',
|
||||||
|
expectedFormat: 'rss',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(0);
|
||||||
|
expect(result.errors).toHaveLength(1);
|
||||||
|
expect(result.errors[0].code).toBe('TIMEOUT');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses body sniff fallback when content-type is missing', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://example.com');
|
||||||
|
mockPool
|
||||||
|
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||||
|
.reply(200, '<?xml version="1.0"?><rss></rss>', {
|
||||||
|
headers: {},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://example.com/feed.xml',
|
||||||
|
expectedFormat: 'rss',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(1);
|
||||||
|
expect(result.errors).toHaveLength(0);
|
||||||
|
expect(result.responses[0].contentType).toBe('application/xml');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts generic application/xml for RSS', async () => {
|
||||||
|
const mockPool = mockAgent.get('https://example.com');
|
||||||
|
mockPool
|
||||||
|
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||||
|
.reply(200, '<?xml version="1.0"?><rss></rss>', {
|
||||||
|
headers: { 'content-type': 'application/xml' },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetcher.fetch({
|
||||||
|
url: 'https://example.com/feed.xml',
|
||||||
|
expectedFormat: 'rss',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(1);
|
||||||
|
expect(result.errors).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('processes multiple feeds with mixed results via fetchMany', async () => {
|
||||||
|
const okPool = mockAgent.get('https://ok.com');
|
||||||
|
okPool
|
||||||
|
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||||
|
.reply(200, '<?xml version="1.0"?><rss></rss>', {
|
||||||
|
headers: { 'content-type': 'application/rss+xml' },
|
||||||
|
});
|
||||||
|
|
||||||
|
const failPool = mockAgent.get('https://fail.com');
|
||||||
|
failPool
|
||||||
|
.intercept({ path: '/feed.xml', method: 'GET' })
|
||||||
|
.reply(500, 'Internal Server Error', {
|
||||||
|
headers: { 'content-type': 'text/plain' },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetcher.fetchMany([
|
||||||
|
{ url: 'https://ok.com/feed.xml', expectedFormat: 'rss' },
|
||||||
|
{ url: 'https://fail.com/feed.xml', expectedFormat: 'rss' },
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(1);
|
||||||
|
expect(result.errors).toHaveLength(1);
|
||||||
|
expect(result.responses[0].source).toBe('https://ok.com/feed.xml');
|
||||||
|
expect(result.errors[0].source).toBe('https://fail.com/feed.xml');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns empty result for empty input array', async () => {
|
||||||
|
const result = await fetcher.fetchMany([]);
|
||||||
|
|
||||||
|
expect(result.responses).toHaveLength(0);
|
||||||
|
expect(result.errors).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
178
modules/fetcher/fetcher.ts
Normal file
178
modules/fetcher/fetcher.ts
Normal file
@ -0,0 +1,178 @@
|
|||||||
|
import { request, type Dispatcher } from 'undici';
|
||||||
|
import type { FetchInput, FetchResult, FeedResponse, FetchError } from '../../interfaces/feed.types.js';
|
||||||
|
import type { IFetcher } from '../../interfaces/fetcher.interface.js';
|
||||||
|
|
||||||
|
const VALID_RSS_CONTENT_TYPES = [
|
||||||
|
'application/xml',
|
||||||
|
'text/xml',
|
||||||
|
'application/rss+xml',
|
||||||
|
];
|
||||||
|
|
||||||
|
const VALID_ATOM_CONTENT_TYPES = [
|
||||||
|
'application/xml',
|
||||||
|
'text/xml',
|
||||||
|
'application/atom+xml',
|
||||||
|
];
|
||||||
|
|
||||||
|
function getValidTypes(expectedFormat: FetchInput['expectedFormat']): string[] {
|
||||||
|
return expectedFormat === 'rss' ? VALID_RSS_CONTENT_TYPES : VALID_ATOM_CONTENT_TYPES;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeContentType(contentType: string): string {
|
||||||
|
return contentType.split(';')[0].trim().toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function isValidFormat(expectedFormat: FetchInput['expectedFormat'], contentType: string): boolean {
|
||||||
|
const normalized = normalizeContentType(contentType);
|
||||||
|
const validTypes = getValidTypes(expectedFormat);
|
||||||
|
return validTypes.includes(normalized);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bodyLooksLikeXml(body: string): boolean {
|
||||||
|
const trimmed = body.trimStart().toLowerCase();
|
||||||
|
return trimmed.startsWith('<?xml') || trimmed.startsWith('<rss') || trimmed.startsWith('<feed');
|
||||||
|
}
|
||||||
|
|
||||||
|
function classifyError(error: unknown, source: string): FetchError {
|
||||||
|
const err = error as Error & { code?: string };
|
||||||
|
const message = err.message ?? String(error);
|
||||||
|
|
||||||
|
if (
|
||||||
|
err.name === 'AbortError' ||
|
||||||
|
err.code === 'UND_ERR_CONNECT_TIMEOUT' ||
|
||||||
|
err.code === 'UND_ERR_REQUEST_TIMEOUT' ||
|
||||||
|
err.code === 'UND_ERR_HEADERS_TIMEOUT'
|
||||||
|
) {
|
||||||
|
return { source, reason: message, code: 'TIMEOUT' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const networkCodes = ['ECONNREFUSED', 'ENOTFOUND', 'ETIMEDOUT', 'EAI_AGAIN', 'ECONNRESET', 'EPIPE', 'ENETUNREACH'];
|
||||||
|
if (
|
||||||
|
networkCodes.includes(err.code ?? '') ||
|
||||||
|
networkCodes.some((code) => message.includes(code))
|
||||||
|
) {
|
||||||
|
return { source, reason: message, code: 'NETWORK' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { source, reason: message, code: 'UNKNOWN' };
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HttpFetcherOptions {
|
||||||
|
timeout?: number;
|
||||||
|
concurrency?: number;
|
||||||
|
dispatcher?: Dispatcher;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class HttpFetcher implements IFetcher {
|
||||||
|
private readonly timeout: number;
|
||||||
|
private readonly concurrency: number;
|
||||||
|
private readonly dispatcher?: Dispatcher;
|
||||||
|
|
||||||
|
constructor(options: HttpFetcherOptions = {}) {
|
||||||
|
this.timeout = options.timeout ?? 10_000;
|
||||||
|
this.concurrency = options.concurrency ?? 5;
|
||||||
|
this.dispatcher = options.dispatcher;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetch(input: FetchInput): Promise<FetchResult> {
|
||||||
|
const result: FetchResult = { responses: [], errors: [], fetchedAt: new Date() };
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { statusCode, headers, body } = await request(input.url, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: { 'User-Agent': 'Pulse-RSS-Fetcher/1.0' },
|
||||||
|
signal: AbortSignal.timeout(this.timeout),
|
||||||
|
dispatcher: this.dispatcher,
|
||||||
|
});
|
||||||
|
|
||||||
|
const responseBody = await body.text();
|
||||||
|
|
||||||
|
if (statusCode < 200 || statusCode >= 300) {
|
||||||
|
result.errors.push({
|
||||||
|
source: input.url,
|
||||||
|
reason: `HTTP ${statusCode}`,
|
||||||
|
code: 'UNKNOWN',
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentType = (headers['content-type'] as string | undefined) ?? '';
|
||||||
|
|
||||||
|
if (contentType) {
|
||||||
|
if (!isValidFormat(input.expectedFormat, contentType)) {
|
||||||
|
if (!bodyLooksLikeXml(responseBody)) {
|
||||||
|
result.errors.push({
|
||||||
|
source: input.url,
|
||||||
|
reason: `Unexpected content type: ${contentType}`,
|
||||||
|
code: 'PARSE',
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (!bodyLooksLikeXml(responseBody)) {
|
||||||
|
result.errors.push({
|
||||||
|
source: input.url,
|
||||||
|
reason: 'Response does not appear to be XML',
|
||||||
|
code: 'PARSE',
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
result.responses.push({
|
||||||
|
source: input.url,
|
||||||
|
body: responseBody,
|
||||||
|
contentType: contentType || 'application/xml',
|
||||||
|
statusCode,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
result.errors.push(classifyError(error, input.url));
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetchMany(inputs: FetchInput[]): Promise<FetchResult> {
|
||||||
|
const result: FetchResult = { responses: [], errors: [], fetchedAt: new Date() };
|
||||||
|
const limit = createConcurrencyLimit(this.concurrency);
|
||||||
|
|
||||||
|
const results = await Promise.all(inputs.map((input) => limit(() => this.fetch(input))));
|
||||||
|
|
||||||
|
for (const r of results) {
|
||||||
|
result.responses.push(...r.responses);
|
||||||
|
result.errors.push(...r.errors);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createConcurrencyLimit(concurrency: number) {
|
||||||
|
const queue: (() => void)[] = [];
|
||||||
|
let activeCount = 0;
|
||||||
|
|
||||||
|
return function <T>(fn: () => Promise<T>): Promise<T> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const run = async () => {
|
||||||
|
activeCount++;
|
||||||
|
try {
|
||||||
|
const result = await fn();
|
||||||
|
resolve(result);
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
} finally {
|
||||||
|
activeCount--;
|
||||||
|
if (queue.length > 0) {
|
||||||
|
const next = queue.shift()!;
|
||||||
|
next();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (activeCount < concurrency) {
|
||||||
|
run();
|
||||||
|
} else {
|
||||||
|
queue.push(run);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
1
modules/fetcher/index.ts
Normal file
1
modules/fetcher/index.ts
Normal file
@ -0,0 +1 @@
|
|||||||
|
export { HttpFetcher, type HttpFetcherOptions } from './fetcher.js';
|
||||||
161
opencode.json
Normal file
161
opencode.json
Normal file
@ -0,0 +1,161 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://opencode.ai/config.json",
|
||||||
|
"agent": {
|
||||||
|
"build": {
|
||||||
|
"mode": "primary",
|
||||||
|
"model": "opencode/kimi-k2.5",
|
||||||
|
"prompt": "{file:./.opencode/prompts/build.txt}",
|
||||||
|
"permission": {
|
||||||
|
"edit": "allow",
|
||||||
|
"bash": {
|
||||||
|
"*": "ask",
|
||||||
|
"npm run *": "allow",
|
||||||
|
"npx vitest *": "allow",
|
||||||
|
"npx tsc *": "allow",
|
||||||
|
"git status": "allow",
|
||||||
|
"git diff *": "allow",
|
||||||
|
"git log *": "allow",
|
||||||
|
"git add *": "ask",
|
||||||
|
"git commit *": "ask",
|
||||||
|
"git push *": "ask"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"plan": {
|
||||||
|
"mode": "primary",
|
||||||
|
"model": "opencode/kimi-k2.5",
|
||||||
|
"temperature": 0.1,
|
||||||
|
"prompt": "{file:./.opencode/prompts/plan.txt}",
|
||||||
|
"permission": {
|
||||||
|
"edit": "deny",
|
||||||
|
"bash": "deny"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"orchestrator": {
|
||||||
|
"description": "Coordinates work across all Pulse modules. Plans tasks, delegates to module agents, owns no business logic.",
|
||||||
|
"mode": "subagent",
|
||||||
|
"model": "opencode/kimi-k2.5",
|
||||||
|
"temperature": 0.1,
|
||||||
|
"prompt": "{file:./.opencode/prompts/orchestrator.txt}",
|
||||||
|
"permission": {
|
||||||
|
"edit": "deny",
|
||||||
|
"bash": "deny",
|
||||||
|
"task": {
|
||||||
|
"*": "allow"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"fetcher-agent": {
|
||||||
|
"description": "Implements and maintains the fetcher module (modules/fetcher/). Handles HTTP fetching of RSS and Atom feeds.",
|
||||||
|
"mode": "subagent",
|
||||||
|
"model": "opencode/kimi-k2.5",
|
||||||
|
"temperature": 0.2,
|
||||||
|
"prompt": "{file:./.opencode/prompts/fetcher.txt}",
|
||||||
|
"permission": {
|
||||||
|
"edit": {
|
||||||
|
"modules/fetcher/**": "allow",
|
||||||
|
"interfaces/**": "ask",
|
||||||
|
"**": "deny"
|
||||||
|
},
|
||||||
|
"bash": {
|
||||||
|
"npx vitest run modules/fetcher*": "allow",
|
||||||
|
"npx tsc --noEmit": "allow",
|
||||||
|
"*": "deny"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser-agent": {
|
||||||
|
"description": "Implements and maintains the parser module (modules/parser/). Converts raw XML/Atom into FeedItem structs.",
|
||||||
|
"mode": "subagent",
|
||||||
|
"model": "opencode/kimi-k2.5",
|
||||||
|
"temperature": 0.2,
|
||||||
|
"prompt": "{file:./.opencode/prompts/parser.txt}",
|
||||||
|
"permission": {
|
||||||
|
"edit": {
|
||||||
|
"modules/parser/**": "allow",
|
||||||
|
"interfaces/**": "ask",
|
||||||
|
"**": "deny"
|
||||||
|
},
|
||||||
|
"bash": {
|
||||||
|
"npx vitest run modules/parser*": "allow",
|
||||||
|
"npx tsc --noEmit": "allow",
|
||||||
|
"*": "deny"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"dedup-agent": {
|
||||||
|
"description": "Implements and maintains the deduplication module (modules/dedup/). Filters already-seen feed items.",
|
||||||
|
"mode": "subagent",
|
||||||
|
"model": "opencode/kimi-k2.5",
|
||||||
|
"temperature": 0.2,
|
||||||
|
"prompt": "{file:./.opencode/prompts/dedup.txt}",
|
||||||
|
"permission": {
|
||||||
|
"edit": {
|
||||||
|
"modules/dedup/**": "allow",
|
||||||
|
"interfaces/**": "ask",
|
||||||
|
"**": "deny"
|
||||||
|
},
|
||||||
|
"bash": {
|
||||||
|
"npx vitest run modules/dedup*": "allow",
|
||||||
|
"npx tsc --noEmit": "allow",
|
||||||
|
"*": "deny"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"storage-agent": {
|
||||||
|
"description": "Implements and maintains the storage module (modules/storage/). Persists FeedItems to SQLite.",
|
||||||
|
"mode": "subagent",
|
||||||
|
"model": "opencode/kimi-k2.5",
|
||||||
|
"temperature": 0.2,
|
||||||
|
"prompt": "{file:./.opencode/prompts/storage.txt}",
|
||||||
|
"permission": {
|
||||||
|
"edit": {
|
||||||
|
"modules/storage/**": "allow",
|
||||||
|
"interfaces/**": "ask",
|
||||||
|
"**": "deny"
|
||||||
|
},
|
||||||
|
"bash": {
|
||||||
|
"npx vitest run modules/storage*": "allow",
|
||||||
|
"npx tsc --noEmit": "allow",
|
||||||
|
"*": "deny"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"formatter-agent": {
|
||||||
|
"description": "Implements and maintains the formatter module (modules/formatter/). Renders FeedItems to terminal, HTML, or JSON.",
|
||||||
|
"mode": "subagent",
|
||||||
|
"model": "opencode/kimi-k2.5",
|
||||||
|
"temperature": 0.2,
|
||||||
|
"prompt": "{file:./.opencode/prompts/formatter.txt}",
|
||||||
|
"permission": {
|
||||||
|
"edit": {
|
||||||
|
"modules/formatter/**": "allow",
|
||||||
|
"interfaces/**": "ask",
|
||||||
|
"**": "deny"
|
||||||
|
},
|
||||||
|
"bash": {
|
||||||
|
"npx vitest run modules/formatter*": "allow",
|
||||||
|
"npx tsc --noEmit": "allow",
|
||||||
|
"*": "deny"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"reviewer": {
|
||||||
|
"description": "Reviews code for quality, correctness, and interface compliance. Read-only.",
|
||||||
|
"mode": "subagent",
|
||||||
|
"model": "opencode/claude-sonnet-4-6",
|
||||||
|
"temperature": 0.1,
|
||||||
|
"prompt": "{file:./.opencode/prompts/reviewer.txt}",
|
||||||
|
"color": "accent",
|
||||||
|
"permission": {
|
||||||
|
"edit": "deny",
|
||||||
|
"bash": {
|
||||||
|
"npx tsc --noEmit": "allow",
|
||||||
|
"npx vitest run": "allow",
|
||||||
|
"git diff *": "allow",
|
||||||
|
"*": "deny"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
1994
package-lock.json
generated
Normal file
1994
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
19
package.json
Normal file
19
package.json
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"name": "pulse",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"description": "RSS feed aggregator",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"test": "vitest run",
|
||||||
|
"typecheck": "tsc --noEmit"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^22.0.0",
|
||||||
|
"tsx": "^4.19.0",
|
||||||
|
"typescript": "^5.7.0",
|
||||||
|
"vitest": "^2.1.0"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"undici": "^6.21.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
20
tsconfig.json
Normal file
20
tsconfig.json
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "NodeNext",
|
||||||
|
"moduleResolution": "NodeNext",
|
||||||
|
"lib": ["ES2022"],
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": ".",
|
||||||
|
"strict": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true
|
||||||
|
},
|
||||||
|
"include": ["**/*.ts"],
|
||||||
|
"exclude": ["node_modules", "dist"]
|
||||||
|
}
|
||||||
8
vitest.config.ts
Normal file
8
vitest.config.ts
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
import { defineConfig } from 'vitest/config';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
test: {
|
||||||
|
globals: true,
|
||||||
|
environment: 'node',
|
||||||
|
},
|
||||||
|
});
|
||||||
Loading…
x
Reference in New Issue
Block a user