Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 185818c

Browse files
soccermaxsoccermax
authored
Fix/admin locks (#349)
* fix get keys for cluster clients * update snapshot * admin service available locks the second * wip * wip * wip * wip * fix tests --------- Co-authored-by: Max Gruenfelder <[email protected]>
1 parent 61d81ce commit 185818c

File tree

8 files changed

+67
-58
lines changed

8 files changed

+67
-58
lines changed

‎package-lock.json‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@cap-js-community/event-queue",
3-
"version": "1.11.0-beta.2",
3+
"version": "1.11.0-beta.3",
44
"description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
55
"main": "src/index.js",
66
"types": "src/index.d.ts",

‎src/config.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const DEFAULT_CHECK_FOR_NEXT_CHUNK = true;
2828
const SUFFIX_PERIODIC = "_PERIODIC";
2929
const CAP_EVENT_TYPE = "CAP_OUTBOX";
3030
const CAP_PARALLEL_DEFAULT = 5;
31+
const CAP_MAX_ATTEMPTS_DEFAULT = 5;
3132
const DELETE_TENANT_BLOCK_AFTER_MS = 5 * 60 * 1000;
3233
const PRIORITIES = Object.values(Priorities);
3334
const UTC_DEFAULT = false;
@@ -387,7 +388,7 @@ class Config {
387388
kind: config.kind ?? "persistent-outbox",
388389
selectMaxChunkSize: config.selectMaxChunkSize ?? config.chunkSize,
389390
parallelEventProcessing: config.parallelEventProcessing ?? (config.parallel && CAP_PARALLEL_DEFAULT),
390-
retryAttempts: config.retryAttempts ?? config.maxAttempts,
391+
retryAttempts: config.retryAttempts ?? config.maxAttempts ?? CAP_MAX_ATTEMPTS_DEFAULT,
391392
...config,
392393
});
393394
eventConfig.internalEvent = true;

‎src/outbox/EventQueueGenericOutboxHandler.js‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
6161
}
6262
} else {
6363
for (const actionName in genericClusterEvents) {
64-
const msg = new cds.Request({
64+
const reg = new cds.Request({
6565
event: EVENT_QUEUE_ACTIONS.CLUSTER,
6666
user: this.context.user,
6767
eventQueue: {
@@ -74,14 +74,14 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
7474
this.#clusterByDataProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
7575
},
7676
});
77-
const clusterResult = await this.__srvUnboxed.tx(this.context).send(msg);
77+
const clusterResult = await this.__srvUnboxed.tx(this.context).send(reg);
7878
if (this.#validateCluster(clusterResult)) {
7979
Object.assign(clusterMap, clusterResult);
8080
} else {
8181
this.logger.error(
8282
"cluster result of handler is not valid. Check the documentation for the expected structure. Continuing without clustering!",
8383
{
84-
handler: msg.event,
84+
handler: reg.event,
8585
clusterResult: JSON.stringify(clusterResult),
8686
}
8787
);
@@ -92,7 +92,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
9292
}
9393

9494
for (const actionName in specificClusterEvents) {
95-
const msg = new cds.Request({
95+
const reg = new cds.Request({
9696
event: `${EVENT_QUEUE_ACTIONS.CLUSTER}.${actionName}`,
9797
user: this.context.user,
9898
eventQueue: {
@@ -105,14 +105,14 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
105105
this.#clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
106106
},
107107
});
108-
const clusterResult = await this.__srvUnboxed.tx(this.context).send(msg);
108+
const clusterResult = await this.__srvUnboxed.tx(this.context).send(reg);
109109
if (this.#validateCluster(clusterResult)) {
110110
Object.assign(clusterMap, clusterResult);
111111
} else {
112112
this.logger.error(
113113
"cluster result of handler is not valid. Check the documentation for the expected structure. Continuing without clustering!",
114114
{
115-
handler: msg.event,
115+
handler: reg.event,
116116
clusterResult: JSON.stringify(clusterResult),
117117
}
118118
);
@@ -264,12 +264,12 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
264264
return payload;
265265
}
266266

267-
const { msg, userId } = this.#buildDispatchData(this.context, payload, {
267+
const { reg, userId } = this.#buildDispatchData(this.context, payload, {
268268
queueEntries: [queueEntry],
269269
});
270-
msg.event = handlerName;
271-
await this.#setContextUser(this.context, userId, msg);
272-
const data = await this.__srvUnboxed.tx(this.context).send(msg);
270+
reg.event = handlerName;
271+
await this.#setContextUser(this.context, userId, reg);
272+
const data = await this.__srvUnboxed.tx(this.context).send(reg);
273273
if (data) {
274274
payload.data = data;
275275
return payload;
@@ -285,12 +285,12 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
285285
return await super.hookForExceededEvents(exceededEvent);
286286
}
287287

288-
const { msg, userId } = this.#buildDispatchData(this.context, exceededEvent.payload, {
288+
const { reg, userId } = this.#buildDispatchData(this.context, exceededEvent.payload, {
289289
queueEntries: [exceededEvent],
290290
});
291-
await this.#setContextUser(this.context, userId, msg);
292-
msg.event = handlerName;
293-
await this.__srvUnboxed.tx(this.context).send(msg);
291+
await this.#setContextUser(this.context, userId, reg);
292+
reg.event = handlerName;
293+
await this.__srvUnboxed.tx(this.context).send(reg);
294294
}
295295

296296
// NOTE: Currently not exposed to CAP service; we wait for a valid use case
@@ -310,37 +310,37 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
310310

311311
async processPeriodicEvent(processContext, key, queueEntry) {
312312
const [, action] = this.eventSubType.split(".");
313-
const msg = new cds.Event({ event: action, eventQueue: { processor: this, key, queueEntries: [queueEntry] } });
314-
await this.#setContextUser(processContext, config.userId, msg);
315-
await this.__srvUnboxed.tx(processContext).emit(msg);
313+
const reg = new cds.Event({ event: action, eventQueue: { processor: this, key, queueEntries: [queueEntry] } });
314+
await this.#setContextUser(processContext, config.userId, reg);
315+
await this.__srvUnboxed.tx(processContext).emit(reg);
316316
}
317317

318318
#buildDispatchData(context, payload, { key, queueEntries } = {}) {
319319
const { useEventQueueUser } = this.eventConfig;
320320
const userId = useEventQueueUser ? config.userId : payload.contextUser;
321-
const msg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
321+
const reg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
322322
const invocationFn = payload._fromSend ? "send" : "emit";
323-
delete msg._fromSend; // TODO: this changes the source object --> check after multiple invocations
324-
delete msg.contextUser;
325-
msg.eventQueue = { processor: this, key, queueEntries, payload };
326-
return { msg, userId, invocationFn };
323+
delete reg._fromSend;
324+
delete reg.contextUser;
325+
reg.eventQueue = { processor: this, key, queueEntries, payload };
326+
return { reg, userId, invocationFn };
327327
}
328328

329-
async #setContextUser(context, userId, data) {
329+
async #setContextUser(context, userId, reg) {
330330
context.user = new cds.User.Privileged({
331331
id: userId,
332332
tokenInfo: await common.getTokenInfo(this.baseContext.tenant),
333333
});
334-
if (data) {
335-
data.user = context.user;
334+
if (reg) {
335+
reg.user = context.user;
336336
}
337337
}
338338

339339
async processEvent(processContext, key, queueEntries, payload) {
340340
try {
341-
const { userId, invocationFn, msg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
342-
await this.#setContextUser(processContext, userId, msg);
343-
const result = await this.__srvUnboxed.tx(processContext)[invocationFn](msg);
341+
const { userId, invocationFn, reg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
342+
await this.#setContextUser(processContext, userId, reg);
343+
const result = await this.__srvUnboxed.tx(processContext)[invocationFn](reg);
344344
return this.#determineResultStatus(result, queueEntries);
345345
} catch (err) {
346346
this.logger.error("error processing outboxed service call", err, {

‎src/shared/distributedLock.js‎

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ const _renewLockRedis = async (context, fullKey, expiryTime, { value = "true" }
108108

109109
const _checkLockExistsRedis = async (context, fullKey) => {
110110
const client = await redis.createMainClientAndConnect(config.redisOptions);
111-
return await client.get(fullKey);
111+
return await client.exists(fullKey);
112112
};
113113

114114
const _checkLockExistsDb = async (context, fullKey) => {
@@ -191,39 +191,36 @@ const _generateKey = (context, tenantScoped, key) => {
191191
};
192192

193193
const getAllLocksRedis = async () => {
194-
const client = await redis.createMainClientAndConnect(config.redisOptions);
195-
const batchSize = 500;
196-
const results = [];
197-
let pipeline = client.multi();
194+
const clientOrCluster = await redis.createMainClientAndConnect(config.redisOptions);
198195
const output = [];
199-
let count = 0;
196+
const results = [];
200197

201-
// NOTE: use SCAN because KEYS is not supported for cluster clients
202-
for await (const key of client.scanIterator({ MATCH: "EVENT*", COUNT: 1000 })) {
203-
const [, tenant, guidOrType, subType] = key.split("##");
204-
if (!subType) {
205-
continue;
206-
}
198+
let clients;
199+
if (redis.isClusterMode()) {
200+
clients = clientOrCluster.masters.map((master) => master.client);
201+
} else {
202+
clients = [clientOrCluster];
203+
}
207204

208-
output.push({
209-
tenant: tenant,
210-
type: guidOrType,
211-
subType: subType,
212-
});
213-
pipeline.ttl(key).get(key);
214-
count++;
205+
// NOTE: use SCAN because KEYS is not supported for cluster clients
206+
for (const client of clients) {
207+
for await (const key of client.scanIterator({ MATCH: "EVENT*", COUNT: 1000 })) {
208+
const [, tenant, guidOrType, subType] = key.split("##");
209+
if (!subType) {
210+
continue;
211+
}
215212

216-
if (count >= batchSize) {
213+
const pipeline = client.multi();
214+
output.push({
215+
tenant: tenant,
216+
type: guidOrType,
217+
subType: subType,
218+
});
219+
pipeline.ttl(key).get(key);
217220
const replies = await pipeline.exec();
218221
results.push(...replies);
219-
pipeline = client.multi();
220-
count = 0;
221222
}
222223
}
223-
if (count > 0) {
224-
const replies = await pipeline.exec();
225-
results.push(...replies);
226-
}
227224

228225
let counter = 0;
229226
for (const row of output) {

‎src/shared/redis.js‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,15 @@ const connectionCheck = async (options) => {
178178
});
179179
};
180180

181+
const isClusterMode = () => {
182+
if (!("__clusterMode" in isClusterMode)) {
183+
const env = getEnvInstance();
184+
const { credentials } = env.redisRequires;
185+
isClusterMode.__clusterMode = credentials.cluster_mode;
186+
}
187+
return isClusterMode.__clusterMode;
188+
};
189+
181190
module.exports = {
182191
createClientAndConnect,
183192
createMainClientAndConnect,
@@ -186,4 +195,5 @@ module.exports = {
186195
closeMainClient,
187196
closeSubscribeClient,
188197
connectionCheck,
198+
isClusterMode,
189199
};

‎test-integration/runner.test.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ describe("runner", () => {
498498
let runTs = await distributedLock.checkLockExistsAndReturnValue({}, runner.__.EVENT_QUEUE_RUN_TS, {
499499
tenantScoped: false,
500500
});
501-
expect(runTs).toBeNull();
501+
expect(runTs).toBeFalsy();
502502
await runner.__._acquireRunId();
503503
runTs = await distributedLock.checkLockExistsAndReturnValue({}, runner.__.EVENT_QUEUE_RUN_TS, {
504504
tenantScoped: false,

‎test/mocks/redisMock.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
let state = {};
44
const _createMainClientAndConnect = async () => ({
55
get: async (key) => state[key]?.value ?? null,
6+
exists: async (key) => Object.prototype.hasOwnProperty.call(state, key),
67
set: async (key, value, options) => {
78
if (state[key]) {
89
return null;

0 commit comments

Comments
 (0)