594 lines
18 KiB
JavaScript
594 lines
18 KiB
JavaScript
|
"use strict";
|
||
|
|
||
|
function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _nonIterableRest(); }
|
||
|
|
||
|
function _iterableToArrayLimit(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
|
||
|
|
||
|
function _toArray(arr) { return _arrayWithHoles(arr) || _iterableToArray(arr) || _nonIterableRest(); }
|
||
|
|
||
|
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance"); }
|
||
|
|
||
|
function _iterableToArray(iter) { if (Symbol.iterator in Object(iter) || Object.prototype.toString.call(iter) === "[object Arguments]") return Array.from(iter); }
|
||
|
|
||
|
function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
|
||
|
|
||
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
|
||
|
|
||
|
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
|
||
|
|
||
|
var Bottleneck,
|
||
|
DEFAULT_PRIORITY,
|
||
|
Events,
|
||
|
Job,
|
||
|
LocalDatastore,
|
||
|
NUM_PRIORITIES,
|
||
|
Queues,
|
||
|
RedisDatastore,
|
||
|
States,
|
||
|
Sync,
|
||
|
parser,
|
||
|
splice = [].splice;
|
||
|
NUM_PRIORITIES = 10;
|
||
|
DEFAULT_PRIORITY = 5;
|
||
|
parser = require("./parser");
|
||
|
Queues = require("./Queues");
|
||
|
Job = require("./Job");
|
||
|
LocalDatastore = require("./LocalDatastore");
|
||
|
RedisDatastore = require("./RedisDatastore");
|
||
|
Events = require("./Events");
|
||
|
States = require("./States");
|
||
|
Sync = require("./Sync");
|
||
|
|
||
|
Bottleneck = function () {
|
||
|
class Bottleneck {
|
||
|
constructor(options = {}, ...invalid) {
|
||
|
var storeInstanceOptions, storeOptions;
|
||
|
this._addToQueue = this._addToQueue.bind(this);
|
||
|
|
||
|
this._validateOptions(options, invalid);
|
||
|
|
||
|
parser.load(options, this.instanceDefaults, this);
|
||
|
this._queues = new Queues(NUM_PRIORITIES);
|
||
|
this._scheduled = {};
|
||
|
this._states = new States(["RECEIVED", "QUEUED", "RUNNING", "EXECUTING"].concat(this.trackDoneStatus ? ["DONE"] : []));
|
||
|
this._limiter = null;
|
||
|
this.Events = new Events(this);
|
||
|
this._submitLock = new Sync("submit", this.Promise);
|
||
|
this._registerLock = new Sync("register", this.Promise);
|
||
|
storeOptions = parser.load(options, this.storeDefaults, {});
|
||
|
|
||
|
this._store = function () {
|
||
|
if (this.datastore === "redis" || this.datastore === "ioredis" || this.connection != null) {
|
||
|
storeInstanceOptions = parser.load(options, this.redisStoreDefaults, {});
|
||
|
return new RedisDatastore(this, storeOptions, storeInstanceOptions);
|
||
|
} else if (this.datastore === "local") {
|
||
|
storeInstanceOptions = parser.load(options, this.localStoreDefaults, {});
|
||
|
return new LocalDatastore(this, storeOptions, storeInstanceOptions);
|
||
|
} else {
|
||
|
throw new Bottleneck.prototype.BottleneckError(`Invalid datastore type: ${this.datastore}`);
|
||
|
}
|
||
|
}.call(this);
|
||
|
|
||
|
this._queues.on("leftzero", () => {
|
||
|
var ref;
|
||
|
return (ref = this._store.heartbeat) != null ? typeof ref.ref === "function" ? ref.ref() : void 0 : void 0;
|
||
|
});
|
||
|
|
||
|
this._queues.on("zero", () => {
|
||
|
var ref;
|
||
|
return (ref = this._store.heartbeat) != null ? typeof ref.unref === "function" ? ref.unref() : void 0 : void 0;
|
||
|
});
|
||
|
}
|
||
|
|
||
|
_validateOptions(options, invalid) {
|
||
|
if (!(options != null && typeof options === "object" && invalid.length === 0)) {
|
||
|
throw new Bottleneck.prototype.BottleneckError("Bottleneck v2 takes a single object argument. Refer to https://github.com/SGrondin/bottleneck#upgrading-to-v2 if you're upgrading from Bottleneck v1.");
|
||
|
}
|
||
|
}
|
||
|
|
||
|
ready() {
|
||
|
return this._store.ready;
|
||
|
}
|
||
|
|
||
|
clients() {
|
||
|
return this._store.clients;
|
||
|
}
|
||
|
|
||
|
channel() {
|
||
|
return `b_${this.id}`;
|
||
|
}
|
||
|
|
||
|
channel_client() {
|
||
|
return `b_${this.id}_${this._store.clientId}`;
|
||
|
}
|
||
|
|
||
|
publish(message) {
|
||
|
return this._store.__publish__(message);
|
||
|
}
|
||
|
|
||
|
disconnect(flush = true) {
|
||
|
return this._store.__disconnect__(flush);
|
||
|
}
|
||
|
|
||
|
chain(_limiter) {
|
||
|
this._limiter = _limiter;
|
||
|
return this;
|
||
|
}
|
||
|
|
||
|
queued(priority) {
|
||
|
return this._queues.queued(priority);
|
||
|
}
|
||
|
|
||
|
clusterQueued() {
|
||
|
return this._store.__queued__();
|
||
|
}
|
||
|
|
||
|
empty() {
|
||
|
return this.queued() === 0 && this._submitLock.isEmpty();
|
||
|
}
|
||
|
|
||
|
running() {
|
||
|
return this._store.__running__();
|
||
|
}
|
||
|
|
||
|
done() {
|
||
|
return this._store.__done__();
|
||
|
}
|
||
|
|
||
|
jobStatus(id) {
|
||
|
return this._states.jobStatus(id);
|
||
|
}
|
||
|
|
||
|
jobs(status) {
|
||
|
return this._states.statusJobs(status);
|
||
|
}
|
||
|
|
||
|
counts() {
|
||
|
return this._states.statusCounts();
|
||
|
}
|
||
|
|
||
|
_randomIndex() {
|
||
|
return Math.random().toString(36).slice(2);
|
||
|
}
|
||
|
|
||
|
check(weight = 1) {
|
||
|
return this._store.__check__(weight);
|
||
|
}
|
||
|
|
||
|
_clearGlobalState(index) {
|
||
|
if (this._scheduled[index] != null) {
|
||
|
clearTimeout(this._scheduled[index].expiration);
|
||
|
delete this._scheduled[index];
|
||
|
return true;
|
||
|
} else {
|
||
|
return false;
|
||
|
}
|
||
|
}
|
||
|
|
||
|
_free(index, job, options, eventInfo) {
|
||
|
var _this = this;
|
||
|
|
||
|
return _asyncToGenerator(function* () {
|
||
|
var e, running;
|
||
|
|
||
|
try {
|
||
|
var _ref = yield _this._store.__free__(index, options.weight);
|
||
|
|
||
|
running = _ref.running;
|
||
|
|
||
|
_this.Events.trigger("debug", `Freed ${options.id}`, eventInfo);
|
||
|
|
||
|
if (running === 0 && _this.empty()) {
|
||
|
return _this.Events.trigger("idle");
|
||
|
}
|
||
|
} catch (error1) {
|
||
|
e = error1;
|
||
|
return _this.Events.trigger("error", e);
|
||
|
}
|
||
|
})();
|
||
|
}
|
||
|
|
||
|
_run(index, job, wait) {
|
||
|
var clearGlobalState, free, run;
|
||
|
job.doRun();
|
||
|
clearGlobalState = this._clearGlobalState.bind(this, index);
|
||
|
run = this._run.bind(this, index, job);
|
||
|
free = this._free.bind(this, index, job);
|
||
|
return this._scheduled[index] = {
|
||
|
timeout: setTimeout(() => {
|
||
|
return job.doExecute(this._limiter, clearGlobalState, run, free);
|
||
|
}, wait),
|
||
|
expiration: job.options.expiration != null ? setTimeout(function () {
|
||
|
return job.doExpire(clearGlobalState, run, free);
|
||
|
}, wait + job.options.expiration) : void 0,
|
||
|
job: job
|
||
|
};
|
||
|
}
|
||
|
|
||
|
_drainOne(capacity) {
|
||
|
return this._registerLock.schedule(() => {
|
||
|
var args, index, next, options, queue;
|
||
|
|
||
|
if (this.queued() === 0) {
|
||
|
return this.Promise.resolve(null);
|
||
|
}
|
||
|
|
||
|
queue = this._queues.getFirst();
|
||
|
|
||
|
var _next2 = next = queue.first();
|
||
|
|
||
|
options = _next2.options;
|
||
|
args = _next2.args;
|
||
|
|
||
|
if (capacity != null && options.weight > capacity) {
|
||
|
return this.Promise.resolve(null);
|
||
|
}
|
||
|
|
||
|
this.Events.trigger("debug", `Draining ${options.id}`, {
|
||
|
args,
|
||
|
options
|
||
|
});
|
||
|
index = this._randomIndex();
|
||
|
return this._store.__register__(index, options.weight, options.expiration).then(({
|
||
|
success,
|
||
|
wait,
|
||
|
reservoir
|
||
|
}) => {
|
||
|
var empty;
|
||
|
this.Events.trigger("debug", `Drained ${options.id}`, {
|
||
|
success,
|
||
|
args,
|
||
|
options
|
||
|
});
|
||
|
|
||
|
if (success) {
|
||
|
queue.shift();
|
||
|
empty = this.empty();
|
||
|
|
||
|
if (empty) {
|
||
|
this.Events.trigger("empty");
|
||
|
}
|
||
|
|
||
|
if (reservoir === 0) {
|
||
|
this.Events.trigger("depleted", empty);
|
||
|
}
|
||
|
|
||
|
this._run(index, next, wait);
|
||
|
|
||
|
return this.Promise.resolve(options.weight);
|
||
|
} else {
|
||
|
return this.Promise.resolve(null);
|
||
|
}
|
||
|
});
|
||
|
});
|
||
|
}
|
||
|
|
||
|
_drainAll(capacity, total = 0) {
|
||
|
return this._drainOne(capacity).then(drained => {
|
||
|
var newCapacity;
|
||
|
|
||
|
if (drained != null) {
|
||
|
newCapacity = capacity != null ? capacity - drained : capacity;
|
||
|
return this._drainAll(newCapacity, total + drained);
|
||
|
} else {
|
||
|
return this.Promise.resolve(total);
|
||
|
}
|
||
|
}).catch(e => {
|
||
|
return this.Events.trigger("error", e);
|
||
|
});
|
||
|
}
|
||
|
|
||
|
_dropAllQueued(message) {
|
||
|
return this._queues.shiftAll(function (job) {
|
||
|
return job.doDrop({
|
||
|
message
|
||
|
});
|
||
|
});
|
||
|
}
|
||
|
|
||
|
stop(options = {}) {
|
||
|
var done, waitForExecuting;
|
||
|
options = parser.load(options, this.stopDefaults);
|
||
|
|
||
|
waitForExecuting = at => {
|
||
|
var finished;
|
||
|
|
||
|
finished = () => {
|
||
|
var counts;
|
||
|
counts = this._states.counts;
|
||
|
return counts[0] + counts[1] + counts[2] + counts[3] === at;
|
||
|
};
|
||
|
|
||
|
return new this.Promise((resolve, reject) => {
|
||
|
if (finished()) {
|
||
|
return resolve();
|
||
|
} else {
|
||
|
return this.on("done", () => {
|
||
|
if (finished()) {
|
||
|
this.removeAllListeners("done");
|
||
|
return resolve();
|
||
|
}
|
||
|
});
|
||
|
}
|
||
|
});
|
||
|
};
|
||
|
|
||
|
done = options.dropWaitingJobs ? (this._run = function (index, next) {
|
||
|
return next.doDrop({
|
||
|
message: options.dropErrorMessage
|
||
|
});
|
||
|
}, this._drainOne = () => {
|
||
|
return this.Promise.resolve(null);
|
||
|
}, this._registerLock.schedule(() => {
|
||
|
return this._submitLock.schedule(() => {
|
||
|
var k, ref, v;
|
||
|
ref = this._scheduled;
|
||
|
|
||
|
for (k in ref) {
|
||
|
v = ref[k];
|
||
|
|
||
|
if (this.jobStatus(v.job.options.id) === "RUNNING") {
|
||
|
clearTimeout(v.timeout);
|
||
|
clearTimeout(v.expiration);
|
||
|
v.job.doDrop({
|
||
|
message: options.dropErrorMessage
|
||
|
});
|
||
|
}
|
||
|
}
|
||
|
|
||
|
this._dropAllQueued(options.dropErrorMessage);
|
||
|
|
||
|
return waitForExecuting(0);
|
||
|
});
|
||
|
})) : this.schedule({
|
||
|
priority: NUM_PRIORITIES - 1,
|
||
|
weight: 0
|
||
|
}, () => {
|
||
|
return waitForExecuting(1);
|
||
|
});
|
||
|
|
||
|
this._receive = function (job) {
|
||
|
return job._reject(new Bottleneck.prototype.BottleneckError(options.enqueueErrorMessage));
|
||
|
};
|
||
|
|
||
|
this.stop = () => {
|
||
|
return this.Promise.reject(new Bottleneck.prototype.BottleneckError("stop() has already been called"));
|
||
|
};
|
||
|
|
||
|
return done;
|
||
|
}
|
||
|
|
||
|
_addToQueue(job) {
|
||
|
var _this2 = this;
|
||
|
|
||
|
return _asyncToGenerator(function* () {
|
||
|
var args, blocked, error, options, reachedHWM, shifted, strategy;
|
||
|
args = job.args;
|
||
|
options = job.options;
|
||
|
|
||
|
try {
|
||
|
var _ref2 = yield _this2._store.__submit__(_this2.queued(), options.weight);
|
||
|
|
||
|
reachedHWM = _ref2.reachedHWM;
|
||
|
blocked = _ref2.blocked;
|
||
|
strategy = _ref2.strategy;
|
||
|
} catch (error1) {
|
||
|
error = error1;
|
||
|
|
||
|
_this2.Events.trigger("debug", `Could not queue ${options.id}`, {
|
||
|
args,
|
||
|
options,
|
||
|
error
|
||
|
});
|
||
|
|
||
|
job.doDrop({
|
||
|
error
|
||
|
});
|
||
|
return false;
|
||
|
}
|
||
|
|
||
|
if (blocked) {
|
||
|
job.doDrop();
|
||
|
return true;
|
||
|
} else if (reachedHWM) {
|
||
|
shifted = strategy === Bottleneck.prototype.strategy.LEAK ? _this2._queues.shiftLastFrom(options.priority) : strategy === Bottleneck.prototype.strategy.OVERFLOW_PRIORITY ? _this2._queues.shiftLastFrom(options.priority + 1) : strategy === Bottleneck.prototype.strategy.OVERFLOW ? job : void 0;
|
||
|
|
||
|
if (shifted != null) {
|
||
|
shifted.doDrop();
|
||
|
}
|
||
|
|
||
|
if (shifted == null || strategy === Bottleneck.prototype.strategy.OVERFLOW) {
|
||
|
if (shifted == null) {
|
||
|
job.doDrop();
|
||
|
}
|
||
|
|
||
|
return reachedHWM;
|
||
|
}
|
||
|
}
|
||
|
|
||
|
job.doQueue(reachedHWM, blocked);
|
||
|
|
||
|
_this2._queues.push(job);
|
||
|
|
||
|
yield _this2._drainAll();
|
||
|
return reachedHWM;
|
||
|
})();
|
||
|
}
|
||
|
|
||
|
_receive(job) {
|
||
|
if (this._states.jobStatus(job.options.id) != null) {
|
||
|
job._reject(new Bottleneck.prototype.BottleneckError(`A job with the same id already exists (id=${job.options.id})`));
|
||
|
|
||
|
return false;
|
||
|
} else {
|
||
|
job.doReceive();
|
||
|
return this._submitLock.schedule(this._addToQueue, job);
|
||
|
}
|
||
|
}
|
||
|
|
||
|
submit(...args) {
|
||
|
var cb, fn, job, options, ref, ref1, task;
|
||
|
|
||
|
if (typeof args[0] === "function") {
|
||
|
var _ref3, _ref4, _splice$call, _splice$call2;
|
||
|
|
||
|
ref = args, (_ref3 = ref, _ref4 = _toArray(_ref3), fn = _ref4[0], args = _ref4.slice(1), _ref3), (_splice$call = splice.call(args, -1), _splice$call2 = _slicedToArray(_splice$call, 1), cb = _splice$call2[0], _splice$call);
|
||
|
options = parser.load({}, this.jobDefaults);
|
||
|
} else {
|
||
|
var _ref5, _ref6, _splice$call3, _splice$call4;
|
||
|
|
||
|
ref1 = args, (_ref5 = ref1, _ref6 = _toArray(_ref5), options = _ref6[0], fn = _ref6[1], args = _ref6.slice(2), _ref5), (_splice$call3 = splice.call(args, -1), _splice$call4 = _slicedToArray(_splice$call3, 1), cb = _splice$call4[0], _splice$call3);
|
||
|
options = parser.load(options, this.jobDefaults);
|
||
|
}
|
||
|
|
||
|
task = (...args) => {
|
||
|
return new this.Promise(function (resolve, reject) {
|
||
|
return fn(...args, function (...args) {
|
||
|
return (args[0] != null ? reject : resolve)(args);
|
||
|
});
|
||
|
});
|
||
|
};
|
||
|
|
||
|
job = new Job(task, args, options, this.jobDefaults, this.rejectOnDrop, this.Events, this._states, this.Promise);
|
||
|
job.promise.then(function (args) {
|
||
|
return typeof cb === "function" ? cb(...args) : void 0;
|
||
|
}).catch(function (args) {
|
||
|
if (Array.isArray(args)) {
|
||
|
return typeof cb === "function" ? cb(...args) : void 0;
|
||
|
} else {
|
||
|
return typeof cb === "function" ? cb(args) : void 0;
|
||
|
}
|
||
|
});
|
||
|
return this._receive(job);
|
||
|
}
|
||
|
|
||
|
schedule(...args) {
|
||
|
var job, options, task;
|
||
|
|
||
|
if (typeof args[0] === "function") {
|
||
|
var _args = args;
|
||
|
|
||
|
var _args2 = _toArray(_args);
|
||
|
|
||
|
task = _args2[0];
|
||
|
args = _args2.slice(1);
|
||
|
options = {};
|
||
|
} else {
|
||
|
var _args3 = args;
|
||
|
|
||
|
var _args4 = _toArray(_args3);
|
||
|
|
||
|
options = _args4[0];
|
||
|
task = _args4[1];
|
||
|
args = _args4.slice(2);
|
||
|
}
|
||
|
|
||
|
job = new Job(task, args, options, this.jobDefaults, this.rejectOnDrop, this.Events, this._states, this.Promise);
|
||
|
|
||
|
this._receive(job);
|
||
|
|
||
|
return job.promise;
|
||
|
}
|
||
|
|
||
|
wrap(fn) {
|
||
|
var schedule, wrapped;
|
||
|
schedule = this.schedule.bind(this);
|
||
|
|
||
|
wrapped = function wrapped(...args) {
|
||
|
return schedule(fn.bind(this), ...args);
|
||
|
};
|
||
|
|
||
|
wrapped.withOptions = function (options, ...args) {
|
||
|
return schedule(options, fn, ...args);
|
||
|
};
|
||
|
|
||
|
return wrapped;
|
||
|
}
|
||
|
|
||
|
updateSettings(options = {}) {
|
||
|
var _this3 = this;
|
||
|
|
||
|
return _asyncToGenerator(function* () {
|
||
|
yield _this3._store.__updateSettings__(parser.overwrite(options, _this3.storeDefaults));
|
||
|
parser.overwrite(options, _this3.instanceDefaults, _this3);
|
||
|
return _this3;
|
||
|
})();
|
||
|
}
|
||
|
|
||
|
currentReservoir() {
|
||
|
return this._store.__currentReservoir__();
|
||
|
}
|
||
|
|
||
|
incrementReservoir(incr = 0) {
|
||
|
return this._store.__incrementReservoir__(incr);
|
||
|
}
|
||
|
|
||
|
}
|
||
|
|
||
|
;
|
||
|
Bottleneck.default = Bottleneck;
|
||
|
Bottleneck.Events = Events;
|
||
|
Bottleneck.version = Bottleneck.prototype.version = require("./version.json").version;
|
||
|
Bottleneck.strategy = Bottleneck.prototype.strategy = {
|
||
|
LEAK: 1,
|
||
|
OVERFLOW: 2,
|
||
|
OVERFLOW_PRIORITY: 4,
|
||
|
BLOCK: 3
|
||
|
};
|
||
|
Bottleneck.BottleneckError = Bottleneck.prototype.BottleneckError = require("./BottleneckError");
|
||
|
Bottleneck.Group = Bottleneck.prototype.Group = require("./Group");
|
||
|
Bottleneck.RedisConnection = Bottleneck.prototype.RedisConnection = require("./RedisConnection");
|
||
|
Bottleneck.IORedisConnection = Bottleneck.prototype.IORedisConnection = require("./IORedisConnection");
|
||
|
Bottleneck.Batcher = Bottleneck.prototype.Batcher = require("./Batcher");
|
||
|
Bottleneck.prototype.jobDefaults = {
|
||
|
priority: DEFAULT_PRIORITY,
|
||
|
weight: 1,
|
||
|
expiration: null,
|
||
|
id: "<no-id>"
|
||
|
};
|
||
|
Bottleneck.prototype.storeDefaults = {
|
||
|
maxConcurrent: null,
|
||
|
minTime: 0,
|
||
|
highWater: null,
|
||
|
strategy: Bottleneck.prototype.strategy.LEAK,
|
||
|
penalty: null,
|
||
|
reservoir: null,
|
||
|
reservoirRefreshInterval: null,
|
||
|
reservoirRefreshAmount: null,
|
||
|
reservoirIncreaseInterval: null,
|
||
|
reservoirIncreaseAmount: null,
|
||
|
reservoirIncreaseMaximum: null
|
||
|
};
|
||
|
Bottleneck.prototype.localStoreDefaults = {
|
||
|
Promise: Promise,
|
||
|
timeout: null,
|
||
|
heartbeatInterval: 250
|
||
|
};
|
||
|
Bottleneck.prototype.redisStoreDefaults = {
|
||
|
Promise: Promise,
|
||
|
timeout: null,
|
||
|
heartbeatInterval: 5000,
|
||
|
clientTimeout: 10000,
|
||
|
Redis: null,
|
||
|
clientOptions: {},
|
||
|
clusterNodes: null,
|
||
|
clearDatastore: false,
|
||
|
connection: null
|
||
|
};
|
||
|
Bottleneck.prototype.instanceDefaults = {
|
||
|
datastore: "local",
|
||
|
connection: null,
|
||
|
id: "<no-id>",
|
||
|
rejectOnDrop: true,
|
||
|
trackDoneStatus: false,
|
||
|
Promise: Promise
|
||
|
};
|
||
|
Bottleneck.prototype.stopDefaults = {
|
||
|
enqueueErrorMessage: "This limiter has been stopped and cannot accept new jobs.",
|
||
|
dropWaitingJobs: true,
|
||
|
dropErrorMessage: "This limiter has been stopped."
|
||
|
};
|
||
|
return Bottleneck;
|
||
|
}.call(void 0);
|
||
|
|
||
|
module.exports = Bottleneck;
|