diff --git a/HISTORY.md b/HISTORY.md index 30f369b5..b15eabce 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,10 @@ +### [1.3.1](https://github.com/bee-queue/bee-queue/compare/v1.3.0...v1.3.1) (2020-11-04) + +### Bug Fixes + +- **scale:** bound unpack arguments count ([#297](https://github.com/bee-queue/bee-queue/issues/297)) ([4108e5e](https://github.com/bee-queue/bee-queue/commit/4108e5e4e97dcf2d8df44dd0114ba31edab511ca)) +- **types:** fix typescript definitions errors ([#311](https://github.com/bee-queue/bee-queue/issues/311)) ([3bc3f31](https://github.com/bee-queue/bee-queue/commit/3bc3f317d09a04b1165fc9f3aa46e17e82d3606f)) + ## [1.3.0](https://github.com/bee-queue/bee-queue/compare/v1.2.3...v1.3.0) (2020-11-03) ### Features diff --git a/index.d.ts b/index.d.ts index 9b0c35d6..e44a430c 100644 --- a/index.d.ts +++ b/index.d.ts @@ -10,7 +10,7 @@ declare class BeeQueue extends EventEmitter { jobs: any; paused: boolean; settings: any; - backoffStrategies: Map number>; + backoffStrategies: Map) => number>; constructor(name: string, settings?: BeeQueue.QueueSettings); @@ -116,10 +116,7 @@ declare namespace BeeQueue { setId(id: string): this; retries(n: number): this; - backoff( - strategy: keyof typeof this.backoffStrategies, - delayFactor?: number - ): this; + backoff(strategy: string, delayFactor?: number): this; delayUntil(dateOrTimestamp: Date | number): this; timeout(milliseconds: number): this; save(): Promise; diff --git a/lib/lua/checkStalledJobs.lua b/lib/lua/checkStalledJobs.lua index 2995bbc5..3bb91f3e 100644 --- a/lib/lua/checkStalledJobs.lua +++ b/lib/lua/checkStalledJobs.lua @@ -12,6 +12,10 @@ if a jobId is not removed from the stalling set within a stallInterval window, we assume the job has stalled and should be reset (moved from active back to waiting) --]] +-- It isn't clear what this value should be, it depends on memory available for the stack. +-- This seems a reasonable limit. +local maxUnpack = 1024 + -- try to update the stallBlock key if not redis.call("set", KEYS[1], "1", "PX", tonumber(ARGV[1]), "NX") then -- hasn't been long enough (stallInterval) since last check @@ -31,20 +35,26 @@ if next(stalling) ~= nil then stalled[#stalled + 1] = jobId end end + + -- lpush instead of rpush so that jobs which cause uncaught exceptions don't + -- hog the job consumers and starve the whole system. not a great situation + -- to be in, but this is fairer. + local pushed = 0 + local nStalled = #stalled -- don't lpush zero jobs (the redis command will fail) - if #stalled > 0 then - -- lpush instead of rpush so that jobs which cause uncaught exceptions don't - -- hog the job consumers and starve the whole system. not a great situation - -- to be in, but this is fairer. - redis.call("lpush", KEYS[3], unpack(stalled)) + while pushed < nStalled do + redis.call("lpush", KEYS[3], unpack(stalled, pushed + 1, math.min(pushed + maxUnpack, nStalled))) + pushed = pushed + maxUnpack end redis.call("del", KEYS[2]) end -- copy currently active jobs into stalling set -local actives = redis.call("lrange", KEYS[4], 0, -1) -if next(actives) ~= nil then - redis.call("sadd", KEYS[2], unpack(actives)) +local actives, added = redis.call("lrange", KEYS[4], 0, -1), 0 +local nActives = #actives +while added < nActives do + redis.call("sadd", KEYS[2], unpack(actives, added + 1, math.min(added + maxUnpack, nActives))) + added = added + maxUnpack end return stalled diff --git a/package-lock.json b/package-lock.json index 580288fa..29471f4e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "bee-queue", - "version": "1.3.0", + "version": "1.3.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 75fb79cc..75a5bbe8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bee-queue", - "version": "1.3.0", + "version": "1.3.1", "description": "A simple, fast, robust job/task queue, backed by Redis.", "main": "index.js", "dependencies": { diff --git a/test/queue-test.js b/test/queue-test.js index 075ae2f4..5f312885 100644 --- a/test/queue-test.js +++ b/test/queue-test.js @@ -1891,6 +1891,36 @@ describe('Queue', (it) => { return done; }); + it('should reset and process more than the unpack limit stalled jobs when starting a queue', async (t) => { + t.plan(0); + + const queue = t.context.makeQueue({ + stallInterval: 1, + }); + + // Create 1025 jobs + const jobs = Array.from(new Array(1025)).map((_, i) => + queue.createJob({foo: `bar${i}`}) + ); + + // Save the jobs. + await Promise.all(jobs.map((job) => job.save())); + + // Artificially move to active. + await queue._getNextJob(); + + // Mark the jobs as stalling, so that Queue#process immediately detects + // them as stalled. + await forceStall(queue); + + const {done, next} = reef(jobs.length); + queue.process(async () => { + next(); + }); + + return done; + }); + it('resets and processes jobs from multiple stalled queues', async (t) => { const queues = []; for (let i = 0; i < 5; i++) {