From 4515180db028206181d6148e63f265863437e6b6 Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Sat, 25 Oct 2025 13:57:55 -0700 Subject: [PATCH 1/3] Improve cron/payouts/aggregate-due-commissions --- .../{ => aggregate-due-commissions}/route.ts | 172 +++++++++++------- apps/web/vercel.json | 8 +- 2 files changed, 106 insertions(+), 74 deletions(-) rename apps/web/app/(ee)/api/cron/payouts/{ => aggregate-due-commissions}/route.ts (56%) diff --git a/apps/web/app/(ee)/api/cron/payouts/route.ts b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts similarity index 56% rename from apps/web/app/(ee)/api/cron/payouts/route.ts rename to apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts index 7acbdcbe911..508c6cd29a5 100644 --- a/apps/web/app/(ee)/api/cron/payouts/route.ts +++ b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts @@ -1,19 +1,38 @@ import { createId } from "@/lib/api/create-id"; import { handleAndReturnErrorResponse } from "@/lib/api/errors"; +import { qstash } from "@/lib/cron"; +import { verifyQstashSignature } from "@/lib/cron/verify-qstash"; import { verifyVercelSignature } from "@/lib/cron/verify-vercel"; import { prisma } from "@dub/prisma"; -import { endOfMonth } from "date-fns"; -import { NextResponse } from "next/server"; +import { APP_DOMAIN_WITH_NGROK, log } from "@dub/utils"; +import { z } from "zod"; +import { logAndRespond } from "../../utils"; export const dynamic = "force-dynamic"; -// This route is used to aggregate pending commissions -// that are past the program holding period into a single payout. -// Runs once every hour (0 * * * *) -// GET /api/cron/payouts -export async function GET(req: Request) { +const BATCH_SIZE = 5000; + +// This cron job aggregates due commissions (pending commissions that are past the program holding period) into payouts. +// Runs once every hour (0 * * * *) + calls itself recursively to look through all pending commissions available. +async function handler(req: Request) { try { - await verifyVercelSignature(req); + let skip: number = 0; + + if (req.method === "GET") { + await verifyVercelSignature(req); + } else if (req.method === "POST") { + const rawBody = await req.text(); + const jsonBody = z + .object({ + skip: z.number(), + }) + .parse(JSON.parse(rawBody)); + skip = jsonBody.skip; + await verifyQstashSignature({ + req, + rawBody, + }); + } const groupedCommissions = await prisma.commission.groupBy({ by: ["programId", "partnerId"], @@ -21,16 +40,21 @@ export async function GET(req: Request) { status: "pending", payoutId: null, }, + skip, + take: BATCH_SIZE, + orderBy: { + partnerId: "asc", + }, }); if (!groupedCommissions.length) { - return NextResponse.json({ - message: "No pending commissions found. Skipping...", - }); + return logAndRespond( + `No partner-program pair with pending commissions found. Skipping...`, + ); } console.log( - `Found ${groupedCommissions.length} pending commissions to process.`, + `Found ${groupedCommissions.length} partner-program pairs with pending commissions to process...`, ); const programIdsToPartnerIds = groupedCommissions.reduce< @@ -58,13 +82,6 @@ export async function GET(req: Request) { select: { name: true, holdingPeriodDays: true, - partners: { - where: { - partnerId: { - in: partnerIds, - }, - }, - }, }, }); @@ -72,41 +89,18 @@ export async function GET(req: Request) { continue; } - const { name, holdingPeriodDays, partners } = program; - - const bannedPartners = partners.filter( - (partner) => partner.status === "banned", - ); - const updatedBannedCommissions = await prisma.commission.updateMany({ + // Find all due commissions for program + const dueCommissionsForProgram = await prisma.commission.findMany({ where: { - earnings: { - gt: 0, - }, + status: "pending", programId, partnerId: { - in: bannedPartners.map(({ partnerId }) => partnerId), + in: partnerIds, }, - status: "pending", - }, - data: { - status: "canceled", - }, - }); - if (updatedBannedCommissions.count > 0) { - console.log( - `Updated ${updatedBannedCommissions.count} banned commissions for program ${name} to canceled`, - ); - } - - // Find all pending commissions - const pendingCommissionsForProgram = await prisma.commission.findMany({ - where: { - status: "pending", - programId, // If there is a holding period set: // we only process commissions that were created before the holding period // but custom commissions are always included - ...(holdingPeriodDays > 0 + ...(program.holdingPeriodDays > 0 ? { OR: [ { @@ -115,7 +109,8 @@ export async function GET(req: Request) { { createdAt: { lt: new Date( - Date.now() - holdingPeriodDays * 24 * 60 * 60 * 1000, + Date.now() - + program.holdingPeriodDays * 24 * 60 * 60 * 1000, ), }, }, @@ -134,12 +129,15 @@ export async function GET(req: Request) { }, }); - if (pendingCommissionsForProgram.length === 0) { + if (dueCommissionsForProgram.length === 0) { + console.log( + `No due commissions found for program ${program.name}, skipping...`, + ); continue; } - const partnerIdsToCommissions = pendingCommissionsForProgram.reduce< - Record + const partnerIdsToCommissions = dueCommissionsForProgram.reduce< + Record >((acc, commission) => { if (!acc[commission.partnerId]) { acc[commission.partnerId] = []; @@ -165,6 +163,11 @@ export async function GET(req: Request) { }, }); + console.log( + `Processing ${partnerIdsToCommissionsArray.length} partners with due commissions for program ${program.name}...`, + ); + let totalProcessed = 0; + for (const { partnerId, commissions } of partnerIdsToCommissionsArray) { // sort the commissions by createdAt const sortedCommissions = commissions.sort( @@ -180,11 +183,9 @@ export async function GET(req: Request) { // earliest commission date const periodStart = sortedCommissions[0].createdAt; - // end of the month of the latest commission date - // e.g. if the latest sale is 2024-12-16, the periodEnd should be 2024-12-31 - const periodEnd = endOfMonth( - sortedCommissions[sortedCommissions.length - 1].createdAt, - ); + // last commission date + const periodEnd = + sortedCommissions[sortedCommissions.length - 1].createdAt; let payoutToUse = existingPendingPayouts.find( (p) => p.partnerId === partnerId, @@ -202,12 +203,10 @@ export async function GET(req: Request) { description: `Dub Partners payout (${program.name})`, }, }); - console.log( - `No existing payout found, created new one ${payoutToUse.id} for partner ${partnerId}`, - ); } - const updatedCommissions = await prisma.commission.updateMany({ + // update the commissions to have the payoutId + await prisma.commission.updateMany({ where: { id: { in: commissions.map((c) => c.id) }, }, @@ -216,13 +215,10 @@ export async function GET(req: Request) { payoutId: payoutToUse.id, }, }); - console.log( - `Updated ${updatedCommissions.count} commissions to have payoutId ${payoutToUse.id}`, - ); // if we're reusing a pending payout, we need to update the amount if (existingPendingPayouts.find((p) => p.id === payoutToUse.id)) { - const updatedPayout = await prisma.payout.update({ + await prisma.payout.update({ where: { id: payoutToUse.id, }, @@ -233,17 +229,53 @@ export async function GET(req: Request) { periodEnd, }, }); - console.log( - `Since we're reusing payout ${payoutToUse.id}, add the new earnings of ${totalEarnings} to the payout amount, making it ${updatedPayout.amount}`, - ); } + + totalProcessed++; } + + const successRate = + (totalProcessed / partnerIdsToCommissionsArray.length) * 100; + console.log( + `Processed ${totalProcessed}/${partnerIdsToCommissionsArray.length} partners with due commissions for program ${program.name} (${successRate.toFixed(1)}% success rate)`, + ); } - return NextResponse.json({ - message: "Commissions payout created.", - }); + if (groupedCommissions.length === BATCH_SIZE) { + const qstashResponse = await qstash.publishJSON({ + url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions`, + body: { + skip: skip + BATCH_SIZE, + }, + }); + if (qstashResponse.messageId) { + console.log( + `Message sent to Qstash with id ${qstashResponse.messageId}`, + ); + } else { + // should never happen, but just in case + await log({ + message: `Error sending message to Qstash to schedule next batch of payouts: ${JSON.stringify(qstashResponse)}`, + type: "errors", + mention: true, + }); + } + return logAndRespond( + `Processed payout commission aggregation for ${groupedCommissions.length} partner-program pairs. Scheduling next batch...`, + ); + } + + return logAndRespond( + `Completed all payout commission aggregation for ${groupedCommissions.length} partner-program pairs.`, + ); } catch (error) { + await log({ + message: `Error aggregating commissions into payouts: ${error.message}`, + type: "errors", + mention: true, + }); return handleAndReturnErrorResponse(error); } } + +export { handler as GET, handler as POST }; diff --git a/apps/web/vercel.json b/apps/web/vercel.json index df97b332d52..b77042440be 100644 --- a/apps/web/vercel.json +++ b/apps/web/vercel.json @@ -32,10 +32,6 @@ "path": "/api/cron/fx-rates", "schedule": "0 8 * * *" }, - { - "path": "/api/cron/payouts", - "schedule": "0 * * * *" - }, { "path": "/api/cron/aggregate-clicks", "schedule": "0 0 * * *" @@ -48,6 +44,10 @@ "path": "/api/cron/partner-program-summary", "schedule": "0 13 1 * *" }, + { + "path": "/api/cron/payouts/aggregate-due-commissions", + "schedule": "0 * * * *" + }, { "path": "/api/cron/payouts/reminders/partners", "schedule": "0 14 * * *" From cd53048cfc5c27999870886e511d72775cef7219 Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Sat, 25 Oct 2025 14:34:44 -0700 Subject: [PATCH 2/3] improve efficiency even more --- .../aggregate-due-commissions/route.ts | 155 ++++++++---------- 1 file changed, 68 insertions(+), 87 deletions(-) diff --git a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts index 508c6cd29a5..dc07e52bbb1 100644 --- a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts +++ b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts @@ -5,102 +5,65 @@ import { verifyQstashSignature } from "@/lib/cron/verify-qstash"; import { verifyVercelSignature } from "@/lib/cron/verify-vercel"; import { prisma } from "@dub/prisma"; import { APP_DOMAIN_WITH_NGROK, log } from "@dub/utils"; -import { z } from "zod"; import { logAndRespond } from "../../utils"; export const dynamic = "force-dynamic"; -const BATCH_SIZE = 5000; +const BATCH_SIZE = 10000; // This cron job aggregates due commissions (pending commissions that are past the program holding period) into payouts. // Runs once every hour (0 * * * *) + calls itself recursively to look through all pending commissions available. async function handler(req: Request) { try { - let skip: number = 0; - if (req.method === "GET") { await verifyVercelSignature(req); } else if (req.method === "POST") { const rawBody = await req.text(); - const jsonBody = z - .object({ - skip: z.number(), - }) - .parse(JSON.parse(rawBody)); - skip = jsonBody.skip; await verifyQstashSignature({ req, rawBody, }); } - const groupedCommissions = await prisma.commission.groupBy({ - by: ["programId", "partnerId"], - where: { - status: "pending", - payoutId: null, + const programsByHoldingPeriod = await prisma.program.groupBy({ + by: ["holdingPeriodDays"], + _count: { + id: true, }, - skip, - take: BATCH_SIZE, orderBy: { - partnerId: "asc", + _count: { + id: "desc", + }, }, }); - if (!groupedCommissions.length) { - return logAndRespond( - `No partner-program pair with pending commissions found. Skipping...`, - ); - } - - console.log( - `Found ${groupedCommissions.length} partner-program pairs with pending commissions to process...`, - ); - - const programIdsToPartnerIds = groupedCommissions.reduce< - Record - >((acc, { programId, partnerId }) => { - acc[programId] ??= []; - if (!acc[programId].includes(partnerId)) { - acc[programId].push(partnerId); - } - return acc; - }, {}); - - const programIdsToPartnerIdsArray = Object.entries( - programIdsToPartnerIds, - ).map(([programId, partnerIds]) => ({ - programId, - partnerIds, - })); - - for (const { programId, partnerIds } of programIdsToPartnerIdsArray) { - const program = await prisma.program.findUnique({ + let holdingPeriodsWithMoreToProcess: number[] = []; + for (const { holdingPeriodDays } of programsByHoldingPeriod) { + const programs = await prisma.program.findMany({ where: { - id: programId, + holdingPeriodDays, }, select: { + id: true, name: true, - holdingPeriodDays: true, }, }); - if (!program) { - continue; - } + console.log( + `Found ${programs.length} programs with holding period days: ${holdingPeriodDays}`, + ); - // Find all due commissions for program - const dueCommissionsForProgram = await prisma.commission.findMany({ + // Find all due commissions (limit by BATCH_SIZE) + const dueCommissions = await prisma.commission.findMany({ where: { status: "pending", - programId, - partnerId: { - in: partnerIds, + programId: { + in: programs.map((p) => p.id), }, - // If there is a holding period set: + // If holding period days is greater than 0: // we only process commissions that were created before the holding period // but custom commissions are always included - ...(program.holdingPeriodDays > 0 + ...(holdingPeriodDays > 0 ? { OR: [ { @@ -109,8 +72,7 @@ async function handler(req: Request) { { createdAt: { lt: new Date( - Date.now() - - program.holdingPeriodDays * 24 * 60 * 60 * 1000, + Date.now() - holdingPeriodDays * 24 * 60 * 60 * 1000, ), }, }, @@ -123,52 +85,70 @@ async function handler(req: Request) { createdAt: true, earnings: true, partnerId: true, + programId: true, }, orderBy: { createdAt: "asc", }, + take: BATCH_SIZE, }); - if (dueCommissionsForProgram.length === 0) { + if (dueCommissions.length === 0) { console.log( - `No due commissions found for program ${program.name}, skipping...`, + `No more due commissions found for programs with holding period days: ${holdingPeriodDays}, skipping...`, ); continue; } - const partnerIdsToCommissions = dueCommissionsForProgram.reduce< - Record + if (dueCommissions.length === BATCH_SIZE) { + holdingPeriodsWithMoreToProcess.push(holdingPeriodDays); + } + + console.log( + `Found ${dueCommissions.length} due commissions for programs with holding period days: ${holdingPeriodDays}`, + ); + + const partnerProgramCommissions = dueCommissions.reduce< + Record >((acc, commission) => { - if (!acc[commission.partnerId]) { - acc[commission.partnerId] = []; + const key = `${commission.partnerId}:${commission.programId}`; + if (!acc[key]) { + acc[key] = []; } - acc[commission.partnerId].push(commission); + acc[key].push(commission); return acc; }, {}); - const partnerIdsToCommissionsArray = Object.entries( - partnerIdsToCommissions, - ).map(([partnerId, commissions]) => ({ - partnerId, + const partnerProgramCommissionsArray = Object.entries( + partnerProgramCommissions, + ).map(([key, commissions]) => ({ + partnerId: key.split(":")[0], + programId: key.split(":")[1], commissions, })); const existingPendingPayouts = await prisma.payout.findMany({ where: { - programId, + programId: { + in: partnerProgramCommissionsArray.map((p) => p.programId), + }, partnerId: { - in: partnerIdsToCommissionsArray.map((p) => p.partnerId), + in: partnerProgramCommissionsArray.map((p) => p.partnerId), }, status: "pending", }, }); console.log( - `Processing ${partnerIdsToCommissionsArray.length} partners with due commissions for program ${program.name}...`, + `Processing ${partnerProgramCommissionsArray.length} partners with due commissions for programs with holding period days: ${holdingPeriodDays}`, ); let totalProcessed = 0; - for (const { partnerId, commissions } of partnerIdsToCommissionsArray) { + for (const { + partnerId, + programId, + commissions, + } of partnerProgramCommissionsArray) { // sort the commissions by createdAt const sortedCommissions = commissions.sort( (a, b) => a.createdAt.getTime() - b.createdAt.getTime(), @@ -200,7 +180,7 @@ async function handler(req: Request) { periodStart, periodEnd, amount: totalEarnings, - description: `Dub Partners payout (${program.name})`, + description: `Dub Partners payout (${programs.find((p) => p.id === programId)?.name})`, }, }); } @@ -216,7 +196,7 @@ async function handler(req: Request) { }, }); - // if we're reusing a pending payout, we need to update the amount + // if we're reusing a pending payout, we need to update the amount and periodEnd if (existingPendingPayouts.find((p) => p.id === payoutToUse.id)) { await prisma.payout.update({ where: { @@ -235,18 +215,19 @@ async function handler(req: Request) { } const successRate = - (totalProcessed / partnerIdsToCommissionsArray.length) * 100; + (totalProcessed / partnerProgramCommissionsArray.length) * 100; console.log( - `Processed ${totalProcessed}/${partnerIdsToCommissionsArray.length} partners with due commissions for program ${program.name} (${successRate.toFixed(1)}% success rate)`, + `Processed ${totalProcessed}/${partnerProgramCommissionsArray.length} partners with due commissions for programs with holding period days: ${holdingPeriodDays} (${successRate.toFixed(1)}% success rate)`, ); } - if (groupedCommissions.length === BATCH_SIZE) { + if (holdingPeriodsWithMoreToProcess.length > 0) { + console.log( + `Several holding periods still have more due commissions: ${holdingPeriodsWithMoreToProcess.join(", ")}`, + ); + const qstashResponse = await qstash.publishJSON({ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions`, - body: { - skip: skip + BATCH_SIZE, - }, }); if (qstashResponse.messageId) { console.log( @@ -261,16 +242,16 @@ async function handler(req: Request) { }); } return logAndRespond( - `Processed payout commission aggregation for ${groupedCommissions.length} partner-program pairs. Scheduling next batch...`, + "Finished aggregating due commissions into payouts for current batch. Scheduling next batch...", ); } return logAndRespond( - `Completed all payout commission aggregation for ${groupedCommissions.length} partner-program pairs.`, + "Finished aggregating due commissions into payouts for all batches.", ); } catch (error) { await log({ - message: `Error aggregating commissions into payouts: ${error.message}`, + message: `Error aggregating due commissions into payouts: ${error.message}`, type: "errors", mention: true, }); From a9ece82d6c131d7e9b47fc25ad053b8768ef0dd0 Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Sat, 25 Oct 2025 14:43:39 -0700 Subject: [PATCH 3/3] address coderabbit feedback --- .../(ee)/api/cron/payouts/aggregate-due-commissions/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts index dc07e52bbb1..08815b0ccd2 100644 --- a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts +++ b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts @@ -168,7 +168,7 @@ async function handler(req: Request) { sortedCommissions[sortedCommissions.length - 1].createdAt; let payoutToUse = existingPendingPayouts.find( - (p) => p.partnerId === partnerId, + (p) => p.partnerId === partnerId && p.programId === programId, ); if (!payoutToUse) {