perf: optimize history sync memory and CPU usage (#385)
* perf: optimize history sync memory and CPU usage Aligns with Baileys upstream PR #2333 . Surgical port of 4 micro-optimizations + 2 test files. No collision with InfiniteAPI's LID/PN, carousel, buttons, Bad MAC or app state sync resilience code. Changes: 1. WAProto longToString/longToNumber: use native BigInt instead of Long library division loops. ~4.6x faster per call (237ns -> 52ns). BigInt.toString() delegates to V8 native C++. Same change applied to fix-imports.js (template) and index.js (generated artifact) so future regenerations stay in sync. 2. downloadHistory: stream-pipe createInflate instead of buffering the full compressed payload then inflating. Cuts RSS peak ~50% on 50MB history payloads. 3. processHistoryMessage: drop the { ...chat } shallow copy before pushing to chats[]. The decoded protobuf object is not referenced after the push. 4. downloadEncryptedContent transform: skip Buffer.concat when remainingBytes is empty (the common case — chunks usually arrive AES-aligned). ~19x faster per chunk (78ms -> 4ms over 25k chunks). Tests: - bigint-validation.test.ts (NEW): 32 parameterized cases covering zero, MAX_SAFE_INTEGER boundary, max int64/uint64, negative values, tricky bit patterns and powers of 2; plus 200-iter fuzz comparing old Long vs new BigInt implementations for equivalence. - proto-tojson-long.test.ts: kept the existing fork-only test for string-in-Long-field graceful handling, appended 4 upstream tests covering Long->string fast path, large unsigned > MAX_SAFE_INTEGER, zero/small values and encode/decode roundtrip.
This commit is contained in:
+11
-11
@@ -1,5 +1,6 @@
|
||||
import { pipeline } from 'stream/promises'
|
||||
import { promisify } from 'util'
|
||||
import { inflate } from 'zlib'
|
||||
import { createInflate, inflate } from 'zlib'
|
||||
import { proto } from '../../WAProto/index.js'
|
||||
import type { Chat, Contact, LIDMapping, WAMessage } from '../Types'
|
||||
import { WAMessageStubType } from '../Types'
|
||||
@@ -29,16 +30,15 @@ const inflatePromise = promisify(inflate)
|
||||
*/
|
||||
export const downloadHistory = async (msg: proto.Message.IHistorySyncNotification, options: RequestInit) => {
|
||||
const stream = await downloadContentFromMessage(msg, 'md-msg-hist', { options })
|
||||
const bufferArray: Buffer[] = []
|
||||
for await (const chunk of stream) {
|
||||
bufferArray.push(chunk)
|
||||
}
|
||||
|
||||
let buffer: Buffer = Buffer.concat(bufferArray)
|
||||
|
||||
// decompress buffer
|
||||
buffer = await inflatePromise(buffer)
|
||||
// Pipe decrypted stream directly through zlib inflate.
|
||||
// Avoids allocating an intermediate buffer for the compressed payload —
|
||||
// memory peaks during 50MB history syncs drop ~50% (PR upstream #2333).
|
||||
const inflater = createInflate()
|
||||
const chunks: Buffer[] = []
|
||||
inflater.on('data', (chunk: Buffer) => chunks.push(chunk))
|
||||
await pipeline(stream, inflater)
|
||||
|
||||
const buffer = Buffer.concat(chunks)
|
||||
const syncData = proto.HistorySync.decode(buffer)
|
||||
return syncData
|
||||
}
|
||||
@@ -361,7 +361,7 @@ export const processHistoryMessage = (item: proto.IHistorySync, logger?: ILogger
|
||||
}
|
||||
}
|
||||
|
||||
chats.push({ ...chat })
|
||||
chats.push(chat)
|
||||
}
|
||||
|
||||
break
|
||||
|
||||
@@ -618,7 +618,7 @@ export const downloadEncryptedContent = async (
|
||||
|
||||
const output = new Transform({
|
||||
transform(chunk, _, callback) {
|
||||
let data = Buffer.concat([remainingBytes, chunk])
|
||||
let data = remainingBytes.length ? Buffer.concat([remainingBytes, chunk]) : chunk
|
||||
|
||||
const decryptLength = toSmallestChunkSize(data.length)
|
||||
remainingBytes = data.slice(decryptLength)
|
||||
|
||||
Reference in New Issue
Block a user