From bf5f4b46496696bde5b50934d37ea97719f34f79 Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Sun, 3 Aug 2025 19:27:06 -0700 Subject: [PATCH 1/2] Improve commissions import efficiency with getLeadEvents --- apps/web/lib/rewardful/import-commissions.ts | 50 ++++++++++++++------ apps/web/lib/tinybird/get-lead-events.ts | 11 +++++ apps/web/scripts/backfill-customers.ts | 9 +--- 3 files changed, 47 insertions(+), 23 deletions(-) create mode 100644 apps/web/lib/tinybird/get-lead-events.ts diff --git a/apps/web/lib/rewardful/import-commissions.ts b/apps/web/lib/rewardful/import-commissions.ts index 20ea4af7845..8f716200aa9 100644 --- a/apps/web/lib/rewardful/import-commissions.ts +++ b/apps/web/lib/rewardful/import-commissions.ts @@ -2,13 +2,14 @@ import { sendEmail } from "@dub/email"; import ProgramImported from "@dub/email/templates/program-imported"; import { prisma } from "@dub/prisma"; import { nanoid } from "@dub/utils"; -import { CommissionStatus, Program } from "@prisma/client"; +import { CommissionStatus, Customer, Link, Program } from "@prisma/client"; import { convertCurrencyWithFxRates } from "../analytics/convert-currency"; import { createId } from "../api/create-id"; import { syncTotalCommissions } from "../api/partners/sync-total-commissions"; -import { getLeadEvent } from "../tinybird"; +import { getLeadEvents } from "../tinybird/get-lead-events"; import { logImportError } from "../tinybird/log-import-error"; import { recordSaleWithTimestamp } from "../tinybird/record-sale"; +import { LeadEventTB } from "../types"; import { redis } from "../upstash"; import { clickEventSchemaTB } from "../zod/schemas/clicks"; import { RewardfulApi } from "./api"; @@ -51,6 +52,23 @@ export async function importCommissions(payload: RewardfulImportPayload) { break; } + const customersData = await prisma.customer.findMany({ + where: { + stripeCustomerId: { + in: commissions + .map((commission) => commission.sale.referral.stripe_customer_id) + .filter(Boolean), + }, + }, + include: { + link: true, + }, + }); + + const customerLeadEvents = await getLeadEvents({ + customerIds: customersData.map((customer) => customer.id), + }).then((res) => res.data); + await Promise.all( commissions.map((commission) => createCommission({ @@ -59,6 +77,8 @@ export async function importCommissions(payload: RewardfulImportPayload) { campaignId, fxRates, importId, + customersData, + customerLeadEvents, }), ), ); @@ -115,12 +135,16 @@ async function createCommission({ campaignId, fxRates, importId, + customersData, + customerLeadEvents, }: { commission: RewardfulCommission; program: Program; campaignId: string; fxRates: Record | null; importId: string; + customersData: (Customer & { link: Link | null })[]; + customerLeadEvents: LeadEventTB[]; }) { const commonImportLogInputs = { workspace_id: program.workspaceId, @@ -223,14 +247,10 @@ async function createCommission({ return; } - const customerFound = await prisma.customer.findUnique({ - where: { - stripeCustomerId: sale.referral.stripe_customer_id, - }, - include: { - link: true, - }, - }); + const customerFound = customersData.find( + (customer) => + customer.stripeCustomerId === sale.referral.stripe_customer_id, + ); if (!customerFound) { await logImportError({ @@ -280,11 +300,11 @@ async function createCommission({ return; } - const leadEvent = await getLeadEvent({ - customerId: customerFound.id, - }); + const leadEvent = customerLeadEvents.find( + (event) => event.customer_id === customerFound.id, + ); - if (!leadEvent || leadEvent.data.length === 0) { + if (!leadEvent) { await logImportError({ ...commonImportLogInputs, entity: "commission", @@ -298,7 +318,7 @@ async function createCommission({ const clickData = clickEventSchemaTB .omit({ timestamp: true }) - .parse(leadEvent.data[0]); + .parse(leadEvent); const eventId = nanoid(16); diff --git a/apps/web/lib/tinybird/get-lead-events.ts b/apps/web/lib/tinybird/get-lead-events.ts new file mode 100644 index 00000000000..8d081edbc05 --- /dev/null +++ b/apps/web/lib/tinybird/get-lead-events.ts @@ -0,0 +1,11 @@ +import z from "../zod"; +import { leadEventSchemaTB } from "../zod/schemas/leads"; +import { tb } from "./client"; + +export const getLeadEvents = tb.buildPipe({ + pipe: "get_lead_events", + parameters: z.object({ + customerIds: z.string().array(), + }), + data: leadEventSchemaTB, +}); diff --git a/apps/web/scripts/backfill-customers.ts b/apps/web/scripts/backfill-customers.ts index 260c5b69073..d412e6292cd 100644 --- a/apps/web/scripts/backfill-customers.ts +++ b/apps/web/scripts/backfill-customers.ts @@ -2,16 +2,9 @@ import { prisma } from "@dub/prisma"; import { Prisma } from "@dub/prisma/client"; import "dotenv-flow/config"; import { tb } from "../lib/tinybird/client"; +import { getLeadEvents } from "../lib/tinybird/get-lead-events"; import z from "../lib/zod"; -export const getLeadEvents = tb.buildPipe({ - pipe: "get_lead_events", - parameters: z.object({ - customerIds: z.array(z.string()), - }), - data: z.any(), -}); - export const getClickEvents = tb.buildPipe({ pipe: "get_click_events", parameters: z.object({ From ac92db0fdc834c08d66d41ada053c78f42fe4e8c Mon Sep 17 00:00:00 2001 From: Steven Tey Date: Sun, 3 Aug 2025 19:41:10 -0700 Subject: [PATCH 2/2] fix import logs link --- .../email/src/templates/program-imported.tsx | 2 +- pipes/get_lead_events.pipe | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 pipes/get_lead_events.pipe diff --git a/packages/email/src/templates/program-imported.tsx b/packages/email/src/templates/program-imported.tsx index 5ef7a433bbe..807cafd87c7 100644 --- a/packages/email/src/templates/program-imported.tsx +++ b/packages/email/src/templates/program-imported.tsx @@ -63,7 +63,7 @@ export default function ProgramImported({ You can{" "} + + % + SELECT * + FROM dub_lead_events + WHERE + true + {% if defined(customerIds) %} AND customer_id IN {{ Array(customerIds, 'String') }} {% end %} + ORDER BY timestamp DESC + +TYPE endpoint + +