diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dbee3965..35b1edd8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,9 @@ jobs: corepack enable corepack prepare yarn@4.x --activate + - name: Fix Git config + run: git config --global core.autocrlf input + - name: Configure Git for HTTPS env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ec3dc44b..0bb1ba60 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,6 +24,9 @@ jobs: corepack enable corepack prepare yarn@4.x --activate + - name: Fix Git config + run: git config --global core.autocrlf input + - name: Configure Git for HTTPS env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e19e434f..83396223 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,9 @@ jobs: corepack enable corepack prepare yarn@4.x --activate + - name: Fix Git config + run: git config --global core.autocrlf input + - name: Configure Git for HTTPS env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/jest.config.ts b/jest.config.ts index 8e9cf43d..d41d8370 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -17,15 +17,26 @@ const config: Config = { useESM: true, tsconfig: { module: 'esnext', + allowJs: true, verbatimModuleSyntax: false, allowImportingTsExtensions: false, }, }, ], - '^.+\\.js$': ['ts-jest', { useESM: true }], + '^.+\\.js$': [ + 'ts-jest', + { + useESM: true, + tsconfig: { + module: 'esnext', + allowJs: true, + verbatimModuleSyntax: false, + }, + }, + ], }, transformIgnorePatterns: [ - 'node_modules/(?!(protobufjs|long|@protobufjs|@types/long)/)', + 'node_modules/(?!(protobufjs|long|@protobufjs|@types/long|whatsapp-rust-bridge)/)', ], } diff --git a/package.json b/package.json index 4d5c23d1..12a5d3f1 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "pino": "^10.3.1", "prom-client": "^15.1.3", "protobufjs": "^8.0.0", - "whatsapp-rust-bridge": "0.5.2", + "whatsapp-rust-bridge": "0.5.3", "ws": "^8.13.0" }, "devDependencies": { diff --git a/src/Socket/chats.ts b/src/Socket/chats.ts index 8ae89566..e02045da 100644 --- a/src/Socket/chats.ts +++ b/src/Socket/chats.ts @@ -37,10 +37,11 @@ import { decodePatches, decodeSyncdSnapshot, encodeSyncdPatch, + ensureLTHashStateVersion, extractSyncdPatches, generateProfilePicture, getHistoryMsg, - ensureLTHashStateVersion, + isAppStateSyncIrrecoverable, newLTHashState, processSyncAction } from '../Utils' @@ -60,7 +61,6 @@ import { } from '../WABinary' import { USyncQuery, USyncUser } from '../WAUSync' import { makeSocket } from './socket.js' -const MAX_SYNC_ATTEMPTS = 2 export const makeChatsSocket = (config: SocketConfig) => { const { @@ -579,6 +579,7 @@ export const makeChatsSocket = (config: SocketConfig) => { // otherwise when we resync from scratch -- all notifications will fire const initialVersionMap: { [T in WAPatchName]?: number } = {} const globalMutationMap: ChatMutationMap = {} + const forceSnapshotCollections = new Set() await authState.keys.transaction(async () => { const collectionsToHandle = new Set(collections) @@ -606,15 +607,20 @@ export const makeChatsSocket = (config: SocketConfig) => { states[name] = state - logger.info(`resyncing ${name} from v${state.version}`) + const shouldForceSnapshot = forceSnapshotCollections.has(name) + if (shouldForceSnapshot) { + forceSnapshotCollections.delete(name) + } + + logger.info(`resyncing ${name} from v${state.version}${shouldForceSnapshot ? ' (forcing snapshot)' : ''}`) nodes.push({ tag: 'collection', attrs: { name, version: state.version.toString(), - // return snapshot if being synced from scratch - return_snapshot: (!state.version).toString() + // return snapshot if syncing from scratch or forcing after a failed attempt + return_snapshot: (shouldForceSnapshot || !state.version).toString() } }) } @@ -685,31 +691,29 @@ export const makeChatsSocket = (config: SocketConfig) => { collectionsToHandle.delete(name) } } catch (error: any) { - // if retry attempts overshoot - // or key not found - const isKeyNotFound = error.output?.statusCode === 404 - const isIrrecoverableError = - (attemptsMap[name] || 0) >= MAX_SYNC_ATTEMPTS || isKeyNotFound || error.name === 'TypeError' - if (isKeyNotFound) { - const currentVersion = states[name]?.version ?? 0 - logger.info( - { name }, - `app state sync: decryption key not available for "${name}" (syncing from v${currentVersion}) -- expected for new sessions where old keys are not shared by the server` - ) - } else { - logger.info( - { name, error: error.stack }, - `failed to sync state from version${isIrrecoverableError ? '' : ', removing and trying from scratch'}` - ) - } - - await authState.keys.set({ 'app-state-sync-version': { [name]: null } }) - // increment number of retries attemptsMap[name] = (attemptsMap[name] || 0) + 1 - if (isIrrecoverableError) { - // stop retrying + const irrecoverable = isAppStateSyncIrrecoverable(error, attemptsMap[name]) + const logData = { + name, + attempt: attemptsMap[name], + version: states[name].version, + statusCode: error.output?.statusCode, + errorType: error.name, + error: error.stack + } + + if (irrecoverable) { + logger.warn(logData, `failed to sync ${name} from v${states[name].version}, giving up`) + // reset persisted version to null so the next resyncAppState call + // requests a full snapshot instead of reusing the stale version that caused the error + await authState.keys.set({ 'app-state-sync-version': { [name]: null } }) collectionsToHandle.delete(name) + } else { + logger.info(logData, `failed to sync ${name} from v${states[name].version}, forcing snapshot retry`) + // force a full snapshot on retry to recover from + // corrupted local state (e.g. LTHash MAC mismatch) + forceSnapshotCollections.add(name) } } } diff --git a/src/Utils/chat-utils.ts b/src/Utils/chat-utils.ts index d131b684..f0afe9ba 100644 --- a/src/Utils/chat-utils.ts +++ b/src/Utils/chat-utils.ts @@ -76,7 +76,7 @@ const to64BitNetworkOrder = (e: number) => { type Mac = { indexMac: Uint8Array; valueMac: Uint8Array; operation: proto.SyncdMutation.SyncdOperation } -const makeLtHashGenerator = ({ indexValueMap, hash }: Pick) => { +export const makeLtHashGenerator = ({ indexValueMap, hash }: Pick) => { indexValueMap = { ...indexValueMap } const addBuffs: Uint8Array[] = [] const subBuffs: Uint8Array[] = [] @@ -87,7 +87,10 @@ const makeLtHashGenerator = ({ indexValueMap, hash }: Pick { return state } +export const MAX_SYNC_ATTEMPTS = 2 + +/** + * Matches WA Web's SyncdFatalError classification: + * XMPP 400/404/405/406 are fatal, TypeError indicates WASM crash. + */ +export const isAppStateSyncIrrecoverable = (error: any, attempts: number): boolean => { + const statusCode = error?.output?.statusCode + return ( + attempts >= MAX_SYNC_ATTEMPTS || + statusCode === 400 || + statusCode === 404 || + statusCode === 405 || + statusCode === 406 || + error?.name === 'TypeError' + ) +} + export const encodeSyncdPatch = async ( { type, index, syncAction, apiVersion, operation }: WAPatchCreate, myAppStateKeyId: string, @@ -492,7 +513,7 @@ export const decodeSyncdSnapshot = async ( const base64Key = Buffer.from(snapKeyId).toString('base64') const keyEnc = await getAppStateSyncKey(base64Key) if (!keyEnc) { - throw new Boom(`failed to find key "${base64Key}" to decode mutation`) + throw new Boom(`failed to find key "${base64Key}" to decode mutation`, { statusCode: 404 }) } const snapKeyData = keyEnc.keyData @@ -581,7 +602,7 @@ export const decodePatches = async ( const base64Key = Buffer.from(patchKeyId).toString('base64') const keyEnc = await getAppStateSyncKey(base64Key) if (!keyEnc) { - throw new Boom(`failed to find key "${base64Key}" to decode mutation`) + throw new Boom(`failed to find key "${base64Key}" to decode mutation`, { statusCode: 404 }) } const patchKeyData = keyEnc.keyData diff --git a/src/__tests__/Utils/app-state-sync-resilience.test.ts b/src/__tests__/Utils/app-state-sync-resilience.test.ts new file mode 100644 index 00000000..ccf9aa18 --- /dev/null +++ b/src/__tests__/Utils/app-state-sync-resilience.test.ts @@ -0,0 +1,98 @@ +import { Boom } from '@hapi/boom' +import { proto } from '../../../WAProto/index.js' +import { + decodeSyncdMutations, + decodeSyncdPatch, + decodeSyncdSnapshot, + isAppStateSyncIrrecoverable, + MAX_SYNC_ATTEMPTS, + newLTHashState +} from '../../Utils/chat-utils' + +const missingKeyFn = async () => null + +describe('App State Sync', () => { + describe('missing key errors throw with statusCode 404', () => { + it('decodeSyncdPatch throws 404 on missing key', async () => { + const msg: proto.ISyncdPatch = { + keyId: { id: Buffer.from('missing-key') }, + mutations: [], + version: { version: 1 as any }, + snapshotMac: Buffer.alloc(32), + patchMac: Buffer.alloc(32) + } + + await expect( + decodeSyncdPatch(msg, 'regular_low', newLTHashState(), missingKeyFn, () => {}, true) + ).rejects.toMatchObject({ output: { statusCode: 404 } }) + }) + + it('decodeSyncdSnapshot throws 404 on missing snapshot key', async () => { + const snapshot: proto.ISyncdSnapshot = { + version: { version: 1 as any }, + records: [], + keyId: { id: Buffer.from('missing-key') }, + mac: Buffer.alloc(32) + } + + await expect(decodeSyncdSnapshot('regular_low', snapshot, missingKeyFn, undefined, true)).rejects.toMatchObject({ + output: { statusCode: 404 } + }) + }) + + it('decodeSyncdMutations throws 404 on missing mutation key', async () => { + const records: proto.ISyncdRecord[] = [ + { + keyId: { id: Buffer.from('missing-key') }, + value: { blob: Buffer.alloc(64) }, + index: { blob: Buffer.alloc(32) } + } + ] + + await expect(decodeSyncdMutations(records, newLTHashState(), missingKeyFn, () => {}, true)).rejects.toMatchObject( + { output: { statusCode: 404 } } + ) + }) + + it('decodeSyncdMutations throws 404 even with validateMacs=false', async () => { + const records: proto.ISyncdRecord[] = [ + { + keyId: { id: Buffer.from('missing-key') }, + value: { blob: Buffer.alloc(64) }, + index: { blob: Buffer.alloc(32) } + } + ] + + await expect( + decodeSyncdMutations(records, newLTHashState(), missingKeyFn, () => {}, false) + ).rejects.toMatchObject({ output: { statusCode: 404 } }) + }) + }) + + describe('isAppStateSyncIrrecoverable', () => { + it.each([400, 404, 405, 406])('should be irrecoverable for status %d on first attempt', statusCode => { + expect(isAppStateSyncIrrecoverable(new Boom('test', { statusCode }), 1)).toBe(true) + }) + + it('should be irrecoverable for TypeError', () => { + expect(isAppStateSyncIrrecoverable(new TypeError('WASM crash'), 1)).toBe(true) + }) + + it('should be irrecoverable when attempts >= MAX_SYNC_ATTEMPTS', () => { + expect(isAppStateSyncIrrecoverable(new Error('generic'), MAX_SYNC_ATTEMPTS)).toBe(true) + }) + + it('should NOT be irrecoverable for generic error below max attempts', () => { + expect(isAppStateSyncIrrecoverable(new Error('generic'), 1)).toBe(false) + }) + + it('should NOT be irrecoverable for non-fatal status codes', () => { + expect(isAppStateSyncIrrecoverable(new Boom('server error', { statusCode: 500 }), 1)).toBe(false) + }) + + it('should handle null/undefined error gracefully', () => { + expect(isAppStateSyncIrrecoverable(null, MAX_SYNC_ATTEMPTS)).toBe(true) + expect(isAppStateSyncIrrecoverable(undefined, 1)).toBe(false) + }) + }) +}) diff --git a/src/__tests__/Utils/app-state-version.test.ts b/src/__tests__/Utils/app-state-version.test.ts new file mode 100644 index 00000000..497fc07f --- /dev/null +++ b/src/__tests__/Utils/app-state-version.test.ts @@ -0,0 +1,64 @@ +import type { LTHashState } from '../../Types' +import { ensureLTHashStateVersion } from '../../Utils/chat-utils' + +describe('ensureLTHashStateVersion', () => { + const makeState = (version: any): LTHashState => ({ + version, + hash: Buffer.alloc(128), + indexValueMap: { someKey: { valueMac: Buffer.from([1, 2, 3]) } } + }) + + it('should return state unchanged for valid numeric version', () => { + const state = makeState(5) + const result = ensureLTHashStateVersion(state) + expect(result).toBe(state) + expect(result.version).toBe(5) + }) + + it('should fix undefined version to 0', () => { + const state = makeState(undefined) + const result = ensureLTHashStateVersion(state) + expect(result.version).toBe(0) + }) + + it('should fix null version to 0', () => { + const state = makeState(null) + const result = ensureLTHashStateVersion(state) + expect(result.version).toBe(0) + }) + + it('should fix NaN version to 0', () => { + const state = makeState(NaN) + const result = ensureLTHashStateVersion(state) + expect(result.version).toBe(0) + }) + + it('should fix string version to 0', () => { + const state = makeState('3' as any) + const result = ensureLTHashStateVersion(state) + expect(result.version).toBe(0) + }) + + it('should keep version 0 as-is', () => { + const state = makeState(0) + const result = ensureLTHashStateVersion(state) + expect(result).toBe(state) + expect(result.version).toBe(0) + }) + + it('should preserve other state fields', () => { + const state = makeState(undefined) + const originalHash = state.hash + const originalMap = state.indexValueMap + const result = ensureLTHashStateVersion(state) + expect(result.hash).toBe(originalHash) + expect(result.indexValueMap).toBe(originalMap) + }) + + it('should allow .toString() after fix without throwing', () => { + const state = makeState(undefined) + ensureLTHashStateVersion(state) + expect(() => state.version.toString()).not.toThrow() + expect(state.version.toString()).toBe('0') + }) +}) diff --git a/src/__tests__/Utils/lt-hash-generator.test.ts b/src/__tests__/Utils/lt-hash-generator.test.ts new file mode 100644 index 00000000..13022e41 --- /dev/null +++ b/src/__tests__/Utils/lt-hash-generator.test.ts @@ -0,0 +1,152 @@ +import { proto } from '../../../WAProto/index.js' +import { makeLtHashGenerator } from '../../Utils/chat-utils' + +const SET = proto.SyncdMutation.SyncdOperation.SET +const REMOVE = proto.SyncdMutation.SyncdOperation.REMOVE + +describe('makeLtHashGenerator', () => { + const makeIndex = (id: string) => Buffer.from(id) + const makeValue = (id: string) => Buffer.from(`value-${id}`) + + it('should handle SET operation normally', () => { + const state = { + hash: Buffer.alloc(128), + indexValueMap: {} + } + + const gen = makeLtHashGenerator(state) + gen.mix({ + indexMac: makeIndex('idx1'), + valueMac: makeValue('v1'), + operation: SET + }) + + const result = gen.finish() + const key = makeIndex('idx1').toString('base64') + expect(result.indexValueMap[key]).toBeDefined() + expect(Buffer.from(result.indexValueMap[key]!.valueMac)).toEqual(makeValue('v1')) + }) + + it('should handle REMOVE with existing previous op', () => { + const indexKey = makeIndex('idx1').toString('base64') + const state = { + hash: Buffer.alloc(128), + indexValueMap: { + [indexKey]: { valueMac: makeValue('old-v1') } + } + } + + const gen = makeLtHashGenerator(state) + gen.mix({ + indexMac: makeIndex('idx1'), + valueMac: makeValue('v1'), + operation: REMOVE + }) + + const result = gen.finish() + expect(result.indexValueMap[indexKey]).toBeUndefined() + }) + + it('should NOT throw on REMOVE without previous op (matches WA Web)', () => { + const state = { + hash: Buffer.alloc(128), + indexValueMap: {} + } + + const gen = makeLtHashGenerator(state) + + expect(() => { + gen.mix({ + indexMac: makeIndex('idx1'), + valueMac: makeValue('v1'), + operation: REMOVE + }) + }).not.toThrow() + + const result = gen.finish() + expect(result.hash).toBeDefined() + expect(result.indexValueMap).toBeDefined() + }) + + it('should continue processing after REMOVE without previous op', () => { + const state = { + hash: Buffer.alloc(128), + indexValueMap: {} + } + + const gen = makeLtHashGenerator(state) + + gen.mix({ + indexMac: makeIndex('idx1'), + valueMac: makeValue('v1'), + operation: REMOVE + }) + + gen.mix({ + indexMac: makeIndex('idx2'), + valueMac: makeValue('v2'), + operation: SET + }) + + const result = gen.finish() + const key2 = makeIndex('idx2').toString('base64') + expect(result.indexValueMap[key2]).toBeDefined() + expect(Buffer.from(result.indexValueMap[key2]!.valueMac)).toEqual(makeValue('v2')) + }) + + it('should not add missing REMOVE to indexValueMap', () => { + const state = { + hash: Buffer.alloc(128), + indexValueMap: {} + } + + const gen = makeLtHashGenerator(state) + gen.mix({ + indexMac: makeIndex('idx1'), + valueMac: makeValue('v1'), + operation: REMOVE + }) + + const result = gen.finish() + const key = makeIndex('idx1').toString('base64') + expect(result.indexValueMap[key]).toBeUndefined() + }) + + it('should process mix of SET, REMOVE-with-prev, and REMOVE-without-prev', () => { + const existingKey = makeIndex('existing').toString('base64') + const state = { + hash: Buffer.alloc(128), + indexValueMap: { + [existingKey]: { valueMac: makeValue('old') } + } + } + + const gen = makeLtHashGenerator(state) + + gen.mix({ + indexMac: makeIndex('new-entry'), + valueMac: makeValue('new'), + operation: SET + }) + + gen.mix({ + indexMac: makeIndex('existing'), + valueMac: makeValue('existing'), + operation: REMOVE + }) + + gen.mix({ + indexMac: makeIndex('ghost'), + valueMac: makeValue('ghost'), + operation: REMOVE + }) + + const result = gen.finish() + const newKey = makeIndex('new-entry').toString('base64') + const ghostKey = makeIndex('ghost').toString('base64') + + expect(result.indexValueMap[newKey]).toBeDefined() + expect(result.indexValueMap[existingKey]).toBeUndefined() + expect(result.indexValueMap[ghostKey]).toBeUndefined() + }) +}) diff --git a/yarn.lock b/yarn.lock index afdfade8..d5dc3f91 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3121,7 +3121,7 @@ __metadata: typedoc: "npm:^0.28.17" typedoc-plugin-markdown: "npm:^4.10.0" typescript: "npm:^5.9.3" - whatsapp-rust-bridge: "npm:0.5.2" + whatsapp-rust-bridge: "npm:0.5.3" ws: "npm:^8.13.0" peerDependencies: audio-decode: ^2.2.3 @@ -8533,10 +8533,10 @@ __metadata: languageName: node linkType: hard -"whatsapp-rust-bridge@npm:0.5.2": - version: 0.5.2 - resolution: "whatsapp-rust-bridge@npm:0.5.2" - checksum: 10c0/022bff4659398144afe6834c279a9fc617a7cbc9177212874150ff2b39019044d2833af85d8ad8d5a40887ad84e7584edead15333e3acce58f989ef7cfd1e6f9 +"whatsapp-rust-bridge@npm:0.5.3": + version: 0.5.3 + resolution: "whatsapp-rust-bridge@npm:0.5.3" + checksum: 10c0/fe9b4c6d6807fd48d51c39b4d8eb41713138b974d01e581626afdb106cfa0ac816782d0a4e3e51164b8f5d70fb55f450a70d224f6a7ed8ddaf9764a0aa1ffbcc languageName: node linkType: hard