78a2b27f6d
Add FeedOrchestrator that coordinates fetch→parse→dedup→store pipeline: - FeedSource type for managing RSS/Atom feed configurations - Feed source CRUD operations in IStorage interface - Database schema migration for feed_sources table - Exponential backoff retry with configurable delays - Per-feed poll intervals with health tracking - Concurrency-limited parallel feed processing - ProcessResult and FeedHealth interfaces for status monitoring Files added: - orchestrator/orchestrator.ts - main orchestrator class - orchestrator/scheduler.ts - backoff calculation utilities - orchestrator/index.ts - module exports - orchestrator/orchestrator.test.ts - comprehensive test suite Files modified: - interfaces/feed.types.ts - add FeedSource type - interfaces/storage.interface.ts - extend with feed source methods - infrastructure/db/database.ts - add FeedSourceTable interface - infrastructure/db/schema.ts - add feed_sources table migration - modules/storage/storage.ts - implement feed source CRUD - modules/storage/storage.test.ts - add feed source tests
89 lines
2.6 KiB
TypeScript
89 lines
2.6 KiB
TypeScript
/**
|
|
* Scheduler utilities for calculating poll delays with exponential backoff.
|
|
*/
|
|
|
|
/**
|
|
* Calculate the next poll delay in milliseconds.
|
|
* Uses exponential backoff based on consecutive failures.
|
|
*
|
|
* Formula: delay = baseDelay * (2 ^ consecutiveFailures) + jitter
|
|
* Capped at maxDelay.
|
|
*
|
|
* @param consecutiveFailures Number of consecutive failed attempts
|
|
* @param baseDelayMs Base delay in milliseconds (e.g., 5000 for 5 seconds)
|
|
* @param maxDelayMs Maximum delay in milliseconds (e.g., 300000 for 5 minutes)
|
|
* @returns Delay in milliseconds before next poll
|
|
*/
|
|
export function calculateNextDelay(
|
|
consecutiveFailures: number,
|
|
baseDelayMs: number,
|
|
maxDelayMs: number
|
|
): number {
|
|
// If no failures, use base delay
|
|
if (consecutiveFailures === 0) {
|
|
return baseDelayMs + calculateJitter(baseDelayMs);
|
|
}
|
|
|
|
// Calculate exponential backoff: base * 2^failures
|
|
// Cap the exponent to avoid overflow
|
|
const cappedFailures = Math.min(consecutiveFailures, 10);
|
|
const exponentialDelay = baseDelayMs * Math.pow(2, cappedFailures);
|
|
|
|
// Apply cap
|
|
const cappedDelay = Math.min(exponentialDelay, maxDelayMs);
|
|
|
|
// Add jitter to prevent thundering herd
|
|
return cappedDelay + calculateJitter(cappedDelay);
|
|
}
|
|
|
|
/**
|
|
* Calculate a small random jitter (±10% of delay).
|
|
* Prevents multiple feeds from syncing up and hitting the server simultaneously.
|
|
*
|
|
* @param delayMs Base delay in milliseconds
|
|
* @returns Jitter offset in milliseconds (can be positive or negative)
|
|
*/
|
|
export function calculateJitter(delayMs: number): number {
|
|
// ±10% jitter, max ±5000ms (5 seconds)
|
|
const maxJitter = Math.min(delayMs * 0.1, 5000);
|
|
return (Math.random() * 2 - 1) * maxJitter;
|
|
}
|
|
|
|
/**
|
|
* Create a concurrency limiter function.
|
|
* Limits the number of promises running concurrently.
|
|
*
|
|
* @param concurrency Maximum number of concurrent operations
|
|
* @returns A function that wraps promises to enforce concurrency limit
|
|
*/
|
|
export function createConcurrencyLimit(concurrency: number) {
|
|
const queue: (() => void)[] = [];
|
|
let activeCount = 0;
|
|
|
|
return function <T>(fn: () => Promise<T>): Promise<T> {
|
|
return new Promise((resolve, reject) => {
|
|
const run = async () => {
|
|
activeCount++;
|
|
try {
|
|
const result = await fn();
|
|
resolve(result);
|
|
} catch (error) {
|
|
reject(error);
|
|
} finally {
|
|
activeCount--;
|
|
if (queue.length > 0) {
|
|
const next = queue.shift()!;
|
|
next();
|
|
}
|
|
}
|
|
};
|
|
|
|
if (activeCount < concurrency) {
|
|
run();
|
|
} else {
|
|
queue.push(run);
|
|
}
|
|
});
|
|
};
|
|
}
|