feat(app-state): App State Sync Resilience (#208)

feat(app-state): App State Sync Resilience (#208)

.github/workflows/build.yml

@@ -24,6 +24,9 @@ jobs:
           corepack enable
           corepack prepare yarn@4.x --activate
 
+      - name: Fix Git config
+        run: git config --global core.autocrlf input
+
       - name: Configure Git for HTTPS
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/lint.yml

@@ -24,6 +24,9 @@ jobs:
           corepack enable
           corepack prepare yarn@4.x --activate
 
+      - name: Fix Git config
+        run: git config --global core.autocrlf input
+
       - name: Configure Git for HTTPS
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -25,6 +25,9 @@ jobs:
           corepack enable
           corepack prepare yarn@4.x --activate
 
+      - name: Fix Git config
+        run: git config --global core.autocrlf input
+
       - name: Configure Git for HTTPS
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jest.config.ts

@@ -17,15 +17,26 @@ const config: Config = {
 				useESM: true,
 				tsconfig: {
 					module: 'esnext',
+					allowJs: true,
 					verbatimModuleSyntax: false,
 					allowImportingTsExtensions: false,
 				},
 			},
 		],
-		'^.+\\.js$': ['ts-jest', { useESM: true }],
+		'^.+\\.js$': [
+			'ts-jest',
+			{
+				useESM: true,
+				tsconfig: {
+					module: 'esnext',
+					allowJs: true,
+					verbatimModuleSyntax: false,
+				},
+			},
+		],
 	},
 	transformIgnorePatterns: [
-		'node_modules/(?!(protobufjs|long|@protobufjs|@types/long)/)',
+		'node_modules/(?!(protobufjs|long|@protobufjs|@types/long|whatsapp-rust-bridge)/)',
 	],
 }
 

package.json

@@ -52,7 +52,7 @@
     "pino": "^10.3.1",
     "prom-client": "^15.1.3",
     "protobufjs": "^8.0.0",
-    "whatsapp-rust-bridge": "0.5.2",
+    "whatsapp-rust-bridge": "0.5.3",
     "ws": "^8.13.0"
   },
   "devDependencies": {

src/Socket/chats.ts

@@ -37,10 +37,11 @@ import {
 	decodePatches,
 	decodeSyncdSnapshot,
 	encodeSyncdPatch,
+	ensureLTHashStateVersion,
 	extractSyncdPatches,
 	generateProfilePicture,
 	getHistoryMsg,
-	ensureLTHashStateVersion,
+	isAppStateSyncIrrecoverable,
 	newLTHashState,
 	processSyncAction
 } from '../Utils'
@@ -60,7 +61,6 @@ import {
 } from '../WABinary'
 import { USyncQuery, USyncUser } from '../WAUSync'
 import { makeSocket } from './socket.js'
-const MAX_SYNC_ATTEMPTS = 2
 
 export const makeChatsSocket = (config: SocketConfig) => {
 	const {
@@ -579,6 +579,7 @@ export const makeChatsSocket = (config: SocketConfig) => {
 			// otherwise when we resync from scratch -- all notifications will fire
 			const initialVersionMap: { [T in WAPatchName]?: number } = {}
 			const globalMutationMap: ChatMutationMap = {}
+			const forceSnapshotCollections = new Set<WAPatchName>()
 
 			await authState.keys.transaction(async () => {
 				const collectionsToHandle = new Set<string>(collections)
@@ -606,15 +607,20 @@ export const makeChatsSocket = (config: SocketConfig) => {
 
 						states[name] = state
 
-						logger.info(`resyncing ${name} from v${state.version}`)
+						const shouldForceSnapshot = forceSnapshotCollections.has(name)
+						if (shouldForceSnapshot) {
+							forceSnapshotCollections.delete(name)
+						}
+
+						logger.info(`resyncing ${name} from v${state.version}${shouldForceSnapshot ? ' (forcing snapshot)' : ''}`)
 
 						nodes.push({
 							tag: 'collection',
 							attrs: {
 								name,
 								version: state.version.toString(),
-								// return snapshot if being synced from scratch
+								// return snapshot if syncing from scratch or forcing after a failed attempt
-								return_snapshot: (!state.version).toString()
+								return_snapshot: (shouldForceSnapshot || !state.version).toString()
 							}
 						})
 					}
@@ -685,31 +691,29 @@ export const makeChatsSocket = (config: SocketConfig) => {
 								collectionsToHandle.delete(name)
 							}
 						} catch (error: any) {
-							// if retry attempts overshoot
-							// or key not found
-							const isKeyNotFound = error.output?.statusCode === 404
-							const isIrrecoverableError =
-								(attemptsMap[name] || 0) >= MAX_SYNC_ATTEMPTS || isKeyNotFound || error.name === 'TypeError'
-							if (isKeyNotFound) {
-								const currentVersion = states[name]?.version ?? 0
-								logger.info(
-									{ name },
-									`app state sync: decryption key not available for "${name}" (syncing from v${currentVersion}) -- expected for new sessions where old keys are not shared by the server`
-								)
-							} else {
-								logger.info(
-									{ name, error: error.stack },
-									`failed to sync state from version${isIrrecoverableError ? '' : ', removing and trying from scratch'}`
-								)
-							}
-
-							await authState.keys.set({ 'app-state-sync-version': { [name]: null } })
-							// increment number of retries
 							attemptsMap[name] = (attemptsMap[name] || 0) + 1
 
-							if (isIrrecoverableError) {
+							const irrecoverable = isAppStateSyncIrrecoverable(error, attemptsMap[name])
-								// stop retrying
+							const logData = {
+								name,
+								attempt: attemptsMap[name],
+								version: states[name].version,
+								statusCode: error.output?.statusCode,
+								errorType: error.name,
+								error: error.stack
+							}
+
+							if (irrecoverable) {
+								logger.warn(logData, `failed to sync ${name} from v${states[name].version}, giving up`)
+								// reset persisted version to null so the next resyncAppState call
+								// requests a full snapshot instead of reusing the stale version that caused the error
+								await authState.keys.set({ 'app-state-sync-version': { [name]: null } })
 								collectionsToHandle.delete(name)
+							} else {
+								logger.info(logData, `failed to sync ${name} from v${states[name].version}, forcing snapshot retry`)
+								// force a full snapshot on retry to recover from
+								// corrupted local state (e.g. LTHash MAC mismatch)
+								forceSnapshotCollections.add(name)
 							}
 						}
 					}

src/Utils/chat-utils.ts

@@ -76,7 +76,7 @@ const to64BitNetworkOrder = (e: number) => {
 
 type Mac = { indexMac: Uint8Array; valueMac: Uint8Array; operation: proto.SyncdMutation.SyncdOperation }
 
-const makeLtHashGenerator = ({ indexValueMap, hash }: Pick<LTHashState, 'hash' | 'indexValueMap'>) => {
+export const makeLtHashGenerator = ({ indexValueMap, hash }: Pick<LTHashState, 'hash' | 'indexValueMap'>) => {
 	indexValueMap = { ...indexValueMap }
 	const addBuffs: Uint8Array[] = []
 	const subBuffs: Uint8Array[] = []
@@ -87,7 +87,10 @@ const makeLtHashGenerator = ({ indexValueMap, hash }: Pick<LTHashState, 'hash' |
 			const prevOp = indexValueMap[indexMacBase64]
 			if (operation === proto.SyncdMutation.SyncdOperation.REMOVE) {
 				if (!prevOp) {
-					throw new Boom('tried remove, but no previous op', { data: { indexMac, valueMac } })
+					// WA Web does not throw here — it logs a warning and skips the subtract.
+					// The missing REMOVE will cause an LTHash mismatch, which is handled
+					// by the MAC validation layer (snapshot recovery or retry).
+					return
 				}
 
 				// remove from index value mac, since this mutation is erased
@@ -138,6 +141,24 @@ export const ensureLTHashStateVersion = (state: LTHashState): LTHashState => {
 	return state
 }
 
+export const MAX_SYNC_ATTEMPTS = 2
+
+/**
+ * Matches WA Web's SyncdFatalError classification:
+ * XMPP 400/404/405/406 are fatal, TypeError indicates WASM crash.
+ */
+export const isAppStateSyncIrrecoverable = (error: any, attempts: number): boolean => {
+	const statusCode = error?.output?.statusCode
+	return (
+		attempts >= MAX_SYNC_ATTEMPTS ||
+		statusCode === 400 ||
+		statusCode === 404 ||
+		statusCode === 405 ||
+		statusCode === 406 ||
+		error?.name === 'TypeError'
+	)
+}
+
 export const encodeSyncdPatch = async (
 	{ type, index, syncAction, apiVersion, operation }: WAPatchCreate,
 	myAppStateKeyId: string,
@@ -492,7 +513,7 @@ export const decodeSyncdSnapshot = async (
 		const base64Key = Buffer.from(snapKeyId).toString('base64')
 		const keyEnc = await getAppStateSyncKey(base64Key)
 		if (!keyEnc) {
-			throw new Boom(`failed to find key "${base64Key}" to decode mutation`)
+			throw new Boom(`failed to find key "${base64Key}" to decode mutation`, { statusCode: 404 })
 		}
 
 		const snapKeyData = keyEnc.keyData
@@ -581,7 +602,7 @@ export const decodePatches = async (
 			const base64Key = Buffer.from(patchKeyId).toString('base64')
 			const keyEnc = await getAppStateSyncKey(base64Key)
 			if (!keyEnc) {
-				throw new Boom(`failed to find key "${base64Key}" to decode mutation`)
+				throw new Boom(`failed to find key "${base64Key}" to decode mutation`, { statusCode: 404 })
 			}
 
 			const patchKeyData = keyEnc.keyData

src/__tests__/Utils/app-state-sync-resilience.test.ts

@@ -0,0 +1,98 @@
+import { Boom } from '@hapi/boom'
+import { proto } from '../../../WAProto/index.js'
+import {
+	decodeSyncdMutations,
+	decodeSyncdPatch,
+	decodeSyncdSnapshot,
+	isAppStateSyncIrrecoverable,
+	MAX_SYNC_ATTEMPTS,
+	newLTHashState
+} from '../../Utils/chat-utils'
+
+const missingKeyFn = async () => null
+
+describe('App State Sync', () => {
+	describe('missing key errors throw with statusCode 404', () => {
+		it('decodeSyncdPatch throws 404 on missing key', async () => {
+			const msg: proto.ISyncdPatch = {
+				keyId: { id: Buffer.from('missing-key') },
+				mutations: [],
+				version: { version: 1 as any },
+				snapshotMac: Buffer.alloc(32),
+				patchMac: Buffer.alloc(32)
+			}
+
+			await expect(
+				decodeSyncdPatch(msg, 'regular_low', newLTHashState(), missingKeyFn, () => {}, true)
+			).rejects.toMatchObject({ output: { statusCode: 404 } })
+		})
+
+		it('decodeSyncdSnapshot throws 404 on missing snapshot key', async () => {
+			const snapshot: proto.ISyncdSnapshot = {
+				version: { version: 1 as any },
+				records: [],
+				keyId: { id: Buffer.from('missing-key') },
+				mac: Buffer.alloc(32)
+			}
+
+			await expect(decodeSyncdSnapshot('regular_low', snapshot, missingKeyFn, undefined, true)).rejects.toMatchObject({
+				output: { statusCode: 404 }
+			})
+		})
+
+		it('decodeSyncdMutations throws 404 on missing mutation key', async () => {
+			const records: proto.ISyncdRecord[] = [
+				{
+					keyId: { id: Buffer.from('missing-key') },
+					value: { blob: Buffer.alloc(64) },
+					index: { blob: Buffer.alloc(32) }
+				}
+			]
+
+			await expect(decodeSyncdMutations(records, newLTHashState(), missingKeyFn, () => {}, true)).rejects.toMatchObject(
+				{ output: { statusCode: 404 } }
+			)
+		})
+
+		it('decodeSyncdMutations throws 404 even with validateMacs=false', async () => {
+			const records: proto.ISyncdRecord[] = [
+				{
+					keyId: { id: Buffer.from('missing-key') },
+					value: { blob: Buffer.alloc(64) },
+					index: { blob: Buffer.alloc(32) }
+				}
+			]
+
+			await expect(
+				decodeSyncdMutations(records, newLTHashState(), missingKeyFn, () => {}, false)
+			).rejects.toMatchObject({ output: { statusCode: 404 } })
+		})
+	})
+
+	describe('isAppStateSyncIrrecoverable', () => {
+		it.each([400, 404, 405, 406])('should be irrecoverable for status %d on first attempt', statusCode => {
+			expect(isAppStateSyncIrrecoverable(new Boom('test', { statusCode }), 1)).toBe(true)
+		})
+
+		it('should be irrecoverable for TypeError', () => {
+			expect(isAppStateSyncIrrecoverable(new TypeError('WASM crash'), 1)).toBe(true)
+		})
+
+		it('should be irrecoverable when attempts >= MAX_SYNC_ATTEMPTS', () => {
+			expect(isAppStateSyncIrrecoverable(new Error('generic'), MAX_SYNC_ATTEMPTS)).toBe(true)
+		})
+
+		it('should NOT be irrecoverable for generic error below max attempts', () => {
+			expect(isAppStateSyncIrrecoverable(new Error('generic'), 1)).toBe(false)
+		})
+
+		it('should NOT be irrecoverable for non-fatal status codes', () => {
+			expect(isAppStateSyncIrrecoverable(new Boom('server error', { statusCode: 500 }), 1)).toBe(false)
+		})
+
+		it('should handle null/undefined error gracefully', () => {
+			expect(isAppStateSyncIrrecoverable(null, MAX_SYNC_ATTEMPTS)).toBe(true)
+			expect(isAppStateSyncIrrecoverable(undefined, 1)).toBe(false)
+		})
+	})
+})

src/__tests__/Utils/app-state-version.test.ts

@@ -0,0 +1,64 @@
+import type { LTHashState } from '../../Types'
+import { ensureLTHashStateVersion } from '../../Utils/chat-utils'
+
+describe('ensureLTHashStateVersion', () => {
+	const makeState = (version: any): LTHashState => ({
+		version,
+		hash: Buffer.alloc(128),
+		indexValueMap: { someKey: { valueMac: Buffer.from([1, 2, 3]) } }
+	})
+
+	it('should return state unchanged for valid numeric version', () => {
+		const state = makeState(5)
+		const result = ensureLTHashStateVersion(state)
+		expect(result).toBe(state)
+		expect(result.version).toBe(5)
+	})
+
+	it('should fix undefined version to 0', () => {
+		const state = makeState(undefined)
+		const result = ensureLTHashStateVersion(state)
+		expect(result.version).toBe(0)
+	})
+
+	it('should fix null version to 0', () => {
+		const state = makeState(null)
+		const result = ensureLTHashStateVersion(state)
+		expect(result.version).toBe(0)
+	})
+
+	it('should fix NaN version to 0', () => {
+		const state = makeState(NaN)
+		const result = ensureLTHashStateVersion(state)
+		expect(result.version).toBe(0)
+	})
+
+	it('should fix string version to 0', () => {
+		const state = makeState('3' as any)
+		const result = ensureLTHashStateVersion(state)
+		expect(result.version).toBe(0)
+	})
+
+	it('should keep version 0 as-is', () => {
+		const state = makeState(0)
+		const result = ensureLTHashStateVersion(state)
+		expect(result).toBe(state)
+		expect(result.version).toBe(0)
+	})
+
+	it('should preserve other state fields', () => {
+		const state = makeState(undefined)
+		const originalHash = state.hash
+		const originalMap = state.indexValueMap
+		const result = ensureLTHashStateVersion(state)
+		expect(result.hash).toBe(originalHash)
+		expect(result.indexValueMap).toBe(originalMap)
+	})
+
+	it('should allow .toString() after fix without throwing', () => {
+		const state = makeState(undefined)
+		ensureLTHashStateVersion(state)
+		expect(() => state.version.toString()).not.toThrow()
+		expect(state.version.toString()).toBe('0')
+	})
+})

src/__tests__/Utils/lt-hash-generator.test.ts

@@ -0,0 +1,152 @@
+import { proto } from '../../../WAProto/index.js'
+import { makeLtHashGenerator } from '../../Utils/chat-utils'
+
+const SET = proto.SyncdMutation.SyncdOperation.SET
+const REMOVE = proto.SyncdMutation.SyncdOperation.REMOVE
+
+describe('makeLtHashGenerator', () => {
+	const makeIndex = (id: string) => Buffer.from(id)
+	const makeValue = (id: string) => Buffer.from(`value-${id}`)
+
+	it('should handle SET operation normally', () => {
+		const state = {
+			hash: Buffer.alloc(128),
+			indexValueMap: {}
+		}
+
+		const gen = makeLtHashGenerator(state)
+		gen.mix({
+			indexMac: makeIndex('idx1'),
+			valueMac: makeValue('v1'),
+			operation: SET
+		})
+
+		const result = gen.finish()
+		const key = makeIndex('idx1').toString('base64')
+		expect(result.indexValueMap[key]).toBeDefined()
+		expect(Buffer.from(result.indexValueMap[key]!.valueMac)).toEqual(makeValue('v1'))
+	})
+
+	it('should handle REMOVE with existing previous op', () => {
+		const indexKey = makeIndex('idx1').toString('base64')
+		const state = {
+			hash: Buffer.alloc(128),
+			indexValueMap: {
+				[indexKey]: { valueMac: makeValue('old-v1') }
+			}
+		}
+
+		const gen = makeLtHashGenerator(state)
+		gen.mix({
+			indexMac: makeIndex('idx1'),
+			valueMac: makeValue('v1'),
+			operation: REMOVE
+		})
+
+		const result = gen.finish()
+		expect(result.indexValueMap[indexKey]).toBeUndefined()
+	})
+
+	it('should NOT throw on REMOVE without previous op (matches WA Web)', () => {
+		const state = {
+			hash: Buffer.alloc(128),
+			indexValueMap: {}
+		}
+
+		const gen = makeLtHashGenerator(state)
+
+		expect(() => {
+			gen.mix({
+				indexMac: makeIndex('idx1'),
+				valueMac: makeValue('v1'),
+				operation: REMOVE
+			})
+		}).not.toThrow()
+
+		const result = gen.finish()
+		expect(result.hash).toBeDefined()
+		expect(result.indexValueMap).toBeDefined()
+	})
+
+	it('should continue processing after REMOVE without previous op', () => {
+		const state = {
+			hash: Buffer.alloc(128),
+			indexValueMap: {}
+		}
+
+		const gen = makeLtHashGenerator(state)
+
+		gen.mix({
+			indexMac: makeIndex('idx1'),
+			valueMac: makeValue('v1'),
+			operation: REMOVE
+		})
+
+		gen.mix({
+			indexMac: makeIndex('idx2'),
+			valueMac: makeValue('v2'),
+			operation: SET
+		})
+
+		const result = gen.finish()
+		const key2 = makeIndex('idx2').toString('base64')
+		expect(result.indexValueMap[key2]).toBeDefined()
+		expect(Buffer.from(result.indexValueMap[key2]!.valueMac)).toEqual(makeValue('v2'))
+	})
+
+	it('should not add missing REMOVE to indexValueMap', () => {
+		const state = {
+			hash: Buffer.alloc(128),
+			indexValueMap: {}
+		}
+
+		const gen = makeLtHashGenerator(state)
+		gen.mix({
+			indexMac: makeIndex('idx1'),
+			valueMac: makeValue('v1'),
+			operation: REMOVE
+		})
+
+		const result = gen.finish()
+		const key = makeIndex('idx1').toString('base64')
+		expect(result.indexValueMap[key]).toBeUndefined()
+	})
+
+	it('should process mix of SET, REMOVE-with-prev, and REMOVE-without-prev', () => {
+		const existingKey = makeIndex('existing').toString('base64')
+		const state = {
+			hash: Buffer.alloc(128),
+			indexValueMap: {
+				[existingKey]: { valueMac: makeValue('old') }
+			}
+		}
+
+		const gen = makeLtHashGenerator(state)
+
+		gen.mix({
+			indexMac: makeIndex('new-entry'),
+			valueMac: makeValue('new'),
+			operation: SET
+		})
+
+		gen.mix({
+			indexMac: makeIndex('existing'),
+			valueMac: makeValue('existing'),
+			operation: REMOVE
+		})
+
+		gen.mix({
+			indexMac: makeIndex('ghost'),
+			valueMac: makeValue('ghost'),
+			operation: REMOVE
+		})
+
+		const result = gen.finish()
+		const newKey = makeIndex('new-entry').toString('base64')
+		const ghostKey = makeIndex('ghost').toString('base64')
+
+		expect(result.indexValueMap[newKey]).toBeDefined()
+		expect(result.indexValueMap[existingKey]).toBeUndefined()
+		expect(result.indexValueMap[ghostKey]).toBeUndefined()
+	})
+})

yarn.lock

@@ -3121,7 +3121,7 @@ __metadata:
     typedoc: "npm:^0.28.17"
     typedoc-plugin-markdown: "npm:^4.10.0"
     typescript: "npm:^5.9.3"
-    whatsapp-rust-bridge: "npm:0.5.2"
+    whatsapp-rust-bridge: "npm:0.5.3"
     ws: "npm:^8.13.0"
   peerDependencies:
     audio-decode: ^2.2.3
@@ -8533,10 +8533,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"whatsapp-rust-bridge@npm:0.5.2":
+"whatsapp-rust-bridge@npm:0.5.3":
-  version: 0.5.2
+  version: 0.5.3
-  resolution: "whatsapp-rust-bridge@npm:0.5.2"
+  resolution: "whatsapp-rust-bridge@npm:0.5.3"
-  checksum: 10c0/022bff4659398144afe6834c279a9fc617a7cbc9177212874150ff2b39019044d2833af85d8ad8d5a40887ad84e7584edead15333e3acce58f989ef7cfd1e6f9
+  checksum: 10c0/fe9b4c6d6807fd48d51c39b4d8eb41713138b974d01e581626afdb106cfa0ac816782d0a4e3e51164b8f5d70fb55f450a70d224f6a7ed8ddaf9764a0aa1ffbcc
   languageName: node
   linkType: hard