From fa2a837a4acefa859f5b4f86d02125b91eb0b9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Lucas?= <55464917+jlucaso1@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:06:15 -0300 Subject: [PATCH] perf: reduce DB calls during sync with caching and batching (#2316) * perf: reduce DB calls during sync with caching and batching * refactor: clean up comments and improve LID-PN mapping storage during history sync --- src/Signal/lid-mapping.ts | 126 ++++++++++++++++++++++++++--------- src/Socket/chats.ts | 18 ++++- src/Types/Events.ts | 1 + src/Utils/process-message.ts | 9 +-- 4 files changed, 113 insertions(+), 41 deletions(-) diff --git a/src/Signal/lid-mapping.ts b/src/Signal/lid-mapping.ts index 2e8b2cd9..75045370 100644 --- a/src/Signal/lid-mapping.ts +++ b/src/Signal/lid-mapping.ts @@ -14,6 +14,9 @@ export class LIDMappingStore { private pnToLIDFunc?: (jids: string[]) => Promise + private readonly inflightLIDLookups = new Map>() + private readonly inflightPNLookups = new Map>() + constructor( keys: SignalKeyStoreWithTransaction, logger: ILogger, @@ -24,12 +27,10 @@ export class LIDMappingStore { this.logger = logger } - /** - * Store LID-PN mapping - USER LEVEL - */ async storeLIDPNMappings(pairs: LIDMapping[]): Promise { - // Validate inputs - const pairMap: { [_: string]: string } = {} + if (pairs.length === 0) return + + const validatedPairs: Array<{ pnUser: string; lidUser: string }> = [] for (const { lid, pn } of pairs) { if (!((isLidUser(lid) && isPnUser(pn)) || (isPnUser(lid) && isLidUser(pn)))) { this.logger.warn(`Invalid LID-PN mapping: ${lid}, ${pn}`) @@ -38,24 +39,43 @@ export class LIDMappingStore { const lidDecoded = jidDecode(lid) const pnDecoded = jidDecode(pn) - if (!lidDecoded || !pnDecoded) continue - const pnUser = pnDecoded.user - const lidUser = lidDecoded.user + validatedPairs.push({ pnUser: pnDecoded.user, lidUser: lidDecoded.user }) + } - let existingLidUser = this.mappingCache.get(`pn:${pnUser}`) - if (!existingLidUser) { - this.logger.trace(`Cache miss for PN user ${pnUser}; checking database`) - const stored = await this.keys.get('lid-mapping', [pnUser]) - existingLidUser = stored[pnUser] + if (validatedPairs.length === 0) return + + const cacheMissSet = new Set() + const existingMappings = new Map() + + for (const { pnUser } of validatedPairs) { + const cached = this.mappingCache.get(`pn:${pnUser}`) + if (cached) { + existingMappings.set(pnUser, cached) + } else { + cacheMissSet.add(pnUser) + } + } + + if (cacheMissSet.size > 0) { + const cacheMisses = [...cacheMissSet] + this.logger.trace(`Batch fetching ${cacheMisses.length} LID mappings from database`) + const stored = await this.keys.get('lid-mapping', cacheMisses) + + for (const pnUser of cacheMisses) { + const existingLidUser = stored[pnUser] if (existingLidUser) { - // Update cache with database value + existingMappings.set(pnUser, existingLidUser) this.mappingCache.set(`pn:${pnUser}`, existingLidUser) this.mappingCache.set(`lid:${existingLidUser}`, pnUser) } } + } + const pairMap: { [_: string]: string } = {} + for (const { pnUser, lidUser } of validatedPairs) { + const existingLidUser = existingMappings.get(pnUser) if (existingLidUser === lidUser) { this.logger.debug({ pnUser, lidUser }, 'LID mapping already exists, skipping') continue @@ -64,33 +84,55 @@ export class LIDMappingStore { pairMap[pnUser] = lidUser } + if (Object.keys(pairMap).length === 0) return + this.logger.trace({ pairMap }, `Storing ${Object.keys(pairMap).length} pn mappings`) - await this.keys.transaction(async () => { - for (const [pnUser, lidUser] of Object.entries(pairMap)) { - await this.keys.set({ - 'lid-mapping': { - [pnUser]: lidUser, - [`${lidUser}_reverse`]: pnUser - } - }) + const batchData: { [key: string]: string } = {} + for (const [pnUser, lidUser] of Object.entries(pairMap)) { + batchData[pnUser] = lidUser + batchData[`${lidUser}_reverse`] = pnUser + } - this.mappingCache.set(`pn:${pnUser}`, lidUser) - this.mappingCache.set(`lid:${lidUser}`, pnUser) - } + await this.keys.transaction(async () => { + await this.keys.set({ 'lid-mapping': batchData }) }, 'lid-mapping') + + // Update cache after successful DB write + for (const [pnUser, lidUser] of Object.entries(pairMap)) { + this.mappingCache.set(`pn:${pnUser}`, lidUser) + this.mappingCache.set(`lid:${lidUser}`, pnUser) + } } - /** - * Get LID for PN - Returns device-specific LID based on user mapping - */ async getLIDForPN(pn: string): Promise { return (await this.getLIDsForPNs([pn]))?.[0]?.lid || null } async getLIDsForPNs(pns: string[]): Promise { + if (pns.length === 0) return null + + const sortedPns = [...new Set(pns)].sort() + const cacheKey = sortedPns.join(',') + + const inflight = this.inflightLIDLookups.get(cacheKey) + if (inflight) { + this.logger.trace(`Coalescing getLIDsForPNs request for ${sortedPns.length} PNs`) + return inflight + } + + const promise = this._getLIDsForPNsImpl(pns) + this.inflightLIDLookups.set(cacheKey, promise) + + try { + return await promise + } finally { + this.inflightLIDLookups.delete(cacheKey) + } + } + + private async _getLIDsForPNsImpl(pns: string[]): Promise { const usyncFetch: { [_: string]: number[] } = {} - // mapped from pn to lid mapping to prevent duplication in results later const successfulPairs: { [_: string]: LIDMapping } = {} const pending: Array<{ pn: string; pnUser: string; decoded: ReturnType }> = [] @@ -118,7 +160,6 @@ export class LIDMappingStore { const decoded = jidDecode(pn) if (!decoded) continue - // Check cache first for PN → LID mapping const pnUser = decoded.user const cached = this.mappingCache.get(`pn:${pnUser}`) if (cached && typeof cached === 'string') { @@ -199,14 +240,33 @@ export class LIDMappingStore { return Object.values(successfulPairs).length > 0 ? Object.values(successfulPairs) : null } - /** - * Get PN for LID - USER LEVEL with device construction - */ async getPNForLID(lid: string): Promise { return (await this.getPNsForLIDs([lid]))?.[0]?.pn || null } async getPNsForLIDs(lids: string[]): Promise { + if (lids.length === 0) return null + + const sortedLids = [...new Set(lids)].sort() + const cacheKey = sortedLids.join(',') + + const inflight = this.inflightPNLookups.get(cacheKey) + if (inflight) { + this.logger.trace(`Coalescing getPNsForLIDs request for ${sortedLids.length} LIDs`) + return inflight + } + + const promise = this._getPNsForLIDsImpl(lids) + this.inflightPNLookups.set(cacheKey, promise) + + try { + return await promise + } finally { + this.inflightPNLookups.delete(cacheKey) + } + } + + private async _getPNsForLIDsImpl(lids: string[]): Promise { const successfulPairs: { [_: string]: LIDMapping } = {} const pending: Array<{ lid: string; lidUser: string; decoded: ReturnType }> = [] diff --git a/src/Socket/chats.ts b/src/Socket/chats.ts index efb38838..7737093a 100644 --- a/src/Socket/chats.ts +++ b/src/Socket/chats.ts @@ -478,6 +478,20 @@ export const makeChatsSocket = (config: SocketConfig) => { const resyncAppState = ev.createBufferedFunction( async (collections: readonly WAPatchName[], isInitialSync: boolean) => { + const appStateSyncKeyCache = new Map() + + const getCachedAppStateSyncKey = async ( + keyId: string + ): Promise => { + if (appStateSyncKeyCache.has(keyId)) { + return appStateSyncKeyCache.get(keyId) ?? undefined + } + + const key = await getAppStateSyncKey(keyId) + appStateSyncKeyCache.set(keyId, key ?? null) + return key + } + // we use this to determine which events to fire // otherwise when we resync from scratch -- all notifications will fire const initialVersionMap: { [T in WAPatchName]?: number } = {} @@ -547,7 +561,7 @@ export const makeChatsSocket = (config: SocketConfig) => { const { state: newState, mutationMap } = await decodeSyncdSnapshot( name, snapshot, - getAppStateSyncKey, + getCachedAppStateSyncKey, initialVersionMap[name], appStateMacVerification.snapshot ) @@ -565,7 +579,7 @@ export const makeChatsSocket = (config: SocketConfig) => { name, patches, states[name], - getAppStateSyncKey, + getCachedAppStateSyncKey, config.options, initialVersionMap[name], logger, diff --git a/src/Types/Events.ts b/src/Types/Events.ts index bf0d1d07..7fb2f337 100644 --- a/src/Types/Events.ts +++ b/src/Types/Events.ts @@ -27,6 +27,7 @@ export type BaileysEventMap = { chats: Chat[] contacts: Contact[] messages: WAMessage[] + lidPnMappings?: LIDMapping[] isLatest?: boolean progress?: number | null syncType?: proto.HistorySync.HistorySyncType | null diff --git a/src/Utils/process-message.ts b/src/Utils/process-message.ts index 2e8a0f40..f57e9127 100644 --- a/src/Utils/process-message.ts +++ b/src/Utils/process-message.ts @@ -287,14 +287,11 @@ const processMessage = async ( const data = await downloadAndProcessHistorySyncNotification(histNotification, options, logger) - // Emit LID-PN mappings from history sync - // This is how WhatsApp Web learns mappings for chats with non-contacts if (data.lidPnMappings?.length) { logger?.debug({ count: data.lidPnMappings.length }, 'processing LID-PN mappings from history sync') - // eslint-disable-next-line max-depth - for (const mapping of data.lidPnMappings) { - ev.emit('lid-mapping.update', mapping) - } + await signalRepository.lidMapping + .storeLIDPNMappings(data.lidPnMappings) + .catch(err => logger?.warn({ err }, 'failed to store LID-PN mappings from history sync')) } ev.emit('messaging-history.set', {