78a2b27f6d
Add FeedOrchestrator that coordinates fetch→parse→dedup→store pipeline: - FeedSource type for managing RSS/Atom feed configurations - Feed source CRUD operations in IStorage interface - Database schema migration for feed_sources table - Exponential backoff retry with configurable delays - Per-feed poll intervals with health tracking - Concurrency-limited parallel feed processing - ProcessResult and FeedHealth interfaces for status monitoring Files added: - orchestrator/orchestrator.ts - main orchestrator class - orchestrator/scheduler.ts - backoff calculation utilities - orchestrator/index.ts - module exports - orchestrator/orchestrator.test.ts - comprehensive test suite Files modified: - interfaces/feed.types.ts - add FeedSource type - interfaces/storage.interface.ts - extend with feed source methods - infrastructure/db/database.ts - add FeedSourceTable interface - infrastructure/db/schema.ts - add feed_sources table migration - modules/storage/storage.ts - implement feed source CRUD - modules/storage/storage.test.ts - add feed source tests
82 lines
3.0 KiB
TypeScript
82 lines
3.0 KiB
TypeScript
/**
|
|
* Database schema migrations.
|
|
* Creates tables idempotently.
|
|
*/
|
|
|
|
import type { Kysely } from 'kysely';
|
|
import type { Database } from './database.js';
|
|
|
|
export async function migrate(db: Kysely<Database>): Promise<void> {
|
|
// Create feed_items table
|
|
await db.schema
|
|
.createTable('feed_items')
|
|
.ifNotExists()
|
|
.addColumn('id', 'varchar(64)', (col) => col.primaryKey())
|
|
.addColumn('source', 'varchar(2048)', (col) => col.notNull())
|
|
.addColumn('title', 'varchar(512)', (col) => col.notNull())
|
|
.addColumn('url', 'varchar(2048)', (col) => col.notNull())
|
|
.addColumn('published_at', 'varchar(32)', (col) => col.notNull())
|
|
.addColumn('content', 'text')
|
|
.addColumn('summary', 'text')
|
|
.addColumn('created_at', 'varchar(32)', (col) => col.notNull().defaultTo('CURRENT_TIMESTAMP'))
|
|
.execute();
|
|
|
|
// Create indexes for feed_items
|
|
await db.schema
|
|
.createIndex('idx_feed_items_source')
|
|
.ifNotExists()
|
|
.on('feed_items')
|
|
.column('source')
|
|
.execute();
|
|
|
|
await db.schema
|
|
.createIndex('idx_feed_items_published')
|
|
.ifNotExists()
|
|
.on('feed_items')
|
|
.column('published_at')
|
|
.execute();
|
|
|
|
// Create seen_ids table
|
|
await db.schema
|
|
.createTable('seen_ids')
|
|
.ifNotExists()
|
|
.addColumn('id', 'varchar(64)', (col) => col.primaryKey())
|
|
.addColumn('seen_at', 'varchar(32)', (col) => col.notNull().defaultTo('CURRENT_TIMESTAMP'))
|
|
.execute();
|
|
|
|
// Create feed_sources table
|
|
await db.schema
|
|
.createTable('feed_sources')
|
|
.ifNotExists()
|
|
.addColumn('id', 'varchar(64)', (col) => col.primaryKey())
|
|
.addColumn('url', 'varchar(2048)', (col) => col.notNull().unique())
|
|
.addColumn('name', 'varchar(256)')
|
|
.addColumn('format', 'varchar(10)', (col) => col.notNull())
|
|
.addColumn('poll_interval_ms', 'integer', (col) => col.notNull())
|
|
.addColumn('is_active', 'boolean', (col) => col.notNull().defaultTo(true))
|
|
.addColumn('last_fetched_at', 'varchar(32)')
|
|
.addColumn('last_success_at', 'varchar(32)')
|
|
.addColumn('consecutive_failures', 'integer', (col) => col.notNull().defaultTo(0))
|
|
.addColumn('created_at', 'varchar(32)', (col) => col.notNull().defaultTo('CURRENT_TIMESTAMP'))
|
|
.addColumn('updated_at', 'varchar(32)', (col) => col.notNull().defaultTo('CURRENT_TIMESTAMP'))
|
|
.execute();
|
|
|
|
// Create index on is_active for quick filtering
|
|
await db.schema
|
|
.createIndex('idx_feed_sources_active')
|
|
.ifNotExists()
|
|
.on('feed_sources')
|
|
.column('is_active')
|
|
.execute();
|
|
}
|
|
|
|
export async function reset(db: Kysely<Database>): Promise<void> {
|
|
// Drop tables (for testing)
|
|
await db.schema.dropIndex('idx_feed_sources_active').ifExists().execute();
|
|
await db.schema.dropTable('feed_sources').ifExists().execute();
|
|
await db.schema.dropTable('seen_ids').ifExists().execute();
|
|
await db.schema.dropIndex('idx_feed_items_published').ifExists().execute();
|
|
await db.schema.dropIndex('idx_feed_items_source').ifExists().execute();
|
|
await db.schema.dropTable('feed_items').ifExists().execute();
|
|
}
|