- commit
- 8672d67
- parent
- e5c3def
- author
- claude@macbookpro
- date
- 2026-03-30 17:38:04 +0800 CST
feat: add jitter to renewal dispatcher to avoid rate limiting Two jitter mechanisms added: - Inter-job jitter (500-3000ms uniform) between consecutive dispatches within a single tick, preventing burst requests to the same platform - Retry backoff jitter (±30% by default) on exponential delay, avoiding thundering herd when multiple jobs retry at the same time All parameters configurable via options: interJobJitterMinMs, interJobJitterMaxMs, retryJitterFactor, random. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 files changed,
+53,
-2
1@@ -22,8 +22,11 @@ import {
2 } from "./projector.js";
3
4 const DEFAULT_EXECUTION_TIMEOUT_MS = 20_000;
5+const DEFAULT_INTER_JOB_JITTER_MAX_MS = 3_000;
6+const DEFAULT_INTER_JOB_JITTER_MIN_MS = 500;
7 const DEFAULT_RECHECK_DELAY_MS = 10_000;
8 const DEFAULT_RETRY_BASE_DELAY_MS = 30_000;
9+const DEFAULT_RETRY_JITTER_FACTOR = 0.3;
10 const DEFAULT_RETRY_MAX_DELAY_MS = 5 * 60_000;
11 const PROXY_DELIVERY_TARGET_KIND = "browser.proxy_delivery";
12 const RUNNER_NAME = "renewal.dispatcher";
13@@ -79,9 +82,13 @@ interface RenewalDispatchOutcome {
14 interface RenewalDispatcherRunnerOptions {
15 browserBridge: BrowserBridgeController | null;
16 executionTimeoutMs?: number;
17+ interJobJitterMaxMs?: number;
18+ interJobJitterMinMs?: number;
19 now?: () => number;
20+ random?: () => number;
21 recheckDelayMs?: number;
22 retryBaseDelayMs?: number;
23+ retryJitterFactor?: number;
24 retryMaxDelayMs?: number;
25 }
26
27@@ -154,8 +161,20 @@ export async function runRenewalDispatcher(
28 let retriedJobs = 0;
29 let skippedJobs = 0;
30 let successfulJobs = 0;
31+ const random = options.random ?? Math.random;
32+ let dispatchedAtLeastOne = false;
33
34 for (const job of dueJobs) {
35+ if (dispatchedAtLeastOne) {
36+ const jitterMs = sampleUniformJitterMs(random, {
37+ maxMs: options.interJobJitterMaxMs,
38+ minMs: options.interJobJitterMinMs
39+ });
40+ if (jitterMs > 0) {
41+ await sleep(jitterMs);
42+ }
43+ }
44+
45 const dispatchContext = await resolveDispatchContext(artifactStore, job);
46
47 if (dispatchContext.conversation == null) {
48@@ -216,7 +235,9 @@ export async function runRenewalDispatcher(
49 errorMessage: dispatchContext.link == null ? "missing_active_link" : "route_unavailable",
50 logDir: context.logDir,
51 now: nowMs,
52+ random,
53 retryBaseDelayMs: options.retryBaseDelayMs,
54+ retryJitterFactor: options.retryJitterFactor,
55 retryMaxDelayMs: options.retryMaxDelayMs,
56 targetSnapshot: dispatchContext.targetSnapshot
57 });
58@@ -271,6 +292,7 @@ export async function runRenewalDispatcher(
59 continue;
60 }
61
62+ dispatchedAtLeastOne = true;
63 const runningJob = await artifactStore.updateRenewalJob({
64 finishedAt: null,
65 jobId: job.jobId,
66@@ -341,7 +363,9 @@ export async function runRenewalDispatcher(
67 errorMessage,
68 logDir: context.logDir,
69 now: now(),
70+ random,
71 retryBaseDelayMs: options.retryBaseDelayMs,
72+ retryJitterFactor: options.retryJitterFactor,
73 retryMaxDelayMs: options.retryMaxDelayMs,
74 targetSnapshot: dispatchContext.targetSnapshot
75 });
76@@ -477,7 +501,9 @@ async function applyFailureOutcome(
77 errorMessage: string;
78 logDir: string | null;
79 now: number;
80+ random: () => number;
81 retryBaseDelayMs?: number;
82+ retryJitterFactor?: number;
83 retryMaxDelayMs?: number;
84 targetSnapshot: RenewalProjectorTargetSnapshot | null;
85 }
86@@ -509,8 +535,9 @@ async function applyFailureOutcome(
87 };
88 }
89
90- const nextAttemptAt = input.now + computeRetryDelayMs(input.attemptCount, {
91+ const nextAttemptAt = input.now + computeRetryDelayMs(input.attemptCount, input.random, {
92 retryBaseDelayMs: input.retryBaseDelayMs,
93+ retryJitterFactor: input.retryJitterFactor,
94 retryMaxDelayMs: input.retryMaxDelayMs
95 });
96 await artifactStore.updateRenewalJob({
97@@ -580,15 +607,21 @@ function buildRenewalPlanId(messageId: string): string {
98
99 function computeRetryDelayMs(
100 attemptCount: number,
101+ random: () => number,
102 options: {
103 retryBaseDelayMs?: number;
104+ retryJitterFactor?: number;
105 retryMaxDelayMs?: number;
106 }
107 ): number {
108 const baseDelayMs = resolvePositiveInteger(options.retryBaseDelayMs, DEFAULT_RETRY_BASE_DELAY_MS);
109 const maxDelayMs = resolvePositiveInteger(options.retryMaxDelayMs, DEFAULT_RETRY_MAX_DELAY_MS);
110+ const jitterFactor = options.retryJitterFactor ?? DEFAULT_RETRY_JITTER_FACTOR;
111 const exponent = Math.max(0, attemptCount - 1);
112- return Math.min(maxDelayMs, baseDelayMs * (2 ** exponent));
113+ const baseDelay = Math.min(maxDelayMs, baseDelayMs * (2 ** exponent));
114+ const jitterRange = baseDelay * Math.min(1, Math.max(0, jitterFactor));
115+ const jitter = Math.round((random() * 2 - 1) * jitterRange);
116+ return Math.max(1, baseDelay + jitter);
117 }
118
119 function normalizeDispatchPayload(
120@@ -792,6 +825,24 @@ function resolvePositiveInteger(value: number | undefined, fallback: number): nu
121 return Number.isInteger(value) && Number(value) > 0 ? Number(value) : fallback;
122 }
123
124+function sleep(ms: number): Promise<void> {
125+ return new Promise((resolve) => {
126+ setTimeout(resolve, ms);
127+ });
128+}
129+
130+function sampleUniformJitterMs(
131+ random: () => number,
132+ options: {
133+ maxMs?: number;
134+ minMs?: number;
135+ }
136+): number {
137+ const minMs = Math.max(0, options.minMs ?? DEFAULT_INTER_JOB_JITTER_MIN_MS);
138+ const maxMs = Math.max(minMs, options.maxMs ?? DEFAULT_INTER_JOB_JITTER_MAX_MS);
139+ return Math.round(minMs + random() * (maxMs - minMs));
140+}
141+
142 function sanitizePathSegment(value: string): string {
143 const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/gu, "-");
144 const collapsed = normalized.replace(/-+/gu, "-").replace(/^-|-$/gu, "");