Skip to content

Commit fae3d42

Browse files
committed
feat: turn sync off for chats the account left
Telegram sends nothing from a channel or group the account has left, so the archive silently fell behind on them: on one account 198 synced chats were missing from the dialog list, some not archived since February. After each comparison with the dialog list, the server asks Telegram about synced channels and groups missing from it, 100 a request, each at most once a day. Sync goes off only when Telegram says the account left or was removed, or the chat is gone or migrated; a chat still joined keeps its sync and is logged, and so is one Telegram gave no answer about. When a chat the server turned off returns to the dialog list, sync comes back on and the gap behind its cursor is filled. Chats the user turned off stay off, and private chats are never asked about: their messages arrive without a dialog. doctor shows both counts.
1 parent 387cc17 commit fae3d42

6 files changed

Lines changed: 440 additions & 18 deletions

File tree

‎cli.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2726,6 +2726,7 @@ async function runDoctor(globalFlags, options = {}) {
27262726
const search = messageSyncService.getSearchStatus();
27272727
const queue = messageSyncService.getQueueStats();
27282728
const repairSuggested = messageSyncService.isRepairSuggested();
2729+
const leftChats = messageSyncService.getLeftChatCounts();
27292730
const update = getUpdateStatus({
27302731
storeDir,
27312732
version: readVersion(),
@@ -2742,6 +2743,7 @@ async function runDoctor(globalFlags, options = {}) {
27422743
ftsVersion: search.version,
27432744
queue,
27442745
repairSuggested,
2746+
leftChats,
27452747
update,
27462748
};
27472749

@@ -2756,6 +2758,12 @@ async function runDoctor(globalFlags, options = {}) {
27562758
console.log(`CONNECTED: ${payload.connected}`);
27572759
console.log(`FTS: ${payload.ftsEnabled}${payload.ftsVersion ? ` (v${payload.ftsVersion})` : ''}`);
27582760
console.log(`QUEUE: pending=${queue.pending} in_progress=${queue.in_progress} idle=${queue.idle} error=${queue.error} gaps=${queue.gaps}`);
2761+
if (leftChats.syncOff) {
2762+
console.log(`LEFT: sync is off for ${leftChats.syncOff} chat(s) you left; it comes back on by itself if you rejoin.`);
2763+
}
2764+
if (leftChats.stillJoined) {
2765+
console.log(`DIALOGS: ${leftChats.stillJoined} synced chat(s) are missing from your dialog list, though Telegram says you are still in them; sync stays on.`);
2766+
}
27592767
if (repairSuggested) {
27602768
console.log(`REPAIR: versions before 2.2.5 could lose messages; fill the holes with \`tgcli backfill repair --all\` (the last ${DEFAULT_REPAIR_SINCE} of every synced chat).`);
27612769
}

‎docs/cli.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ it. `server restart` switches to the caller's version in any direction. See
147147
and the command that installs it, `up to date` with the time that version was
148148
last confirmed, `not checked yet`, or why there is no check, plus the error of a
149149
failed last check. With `--json` the same facts are under `update`.
150+
- `LEFT:` counts chats whose sync the server turned off because you left them (see
151+
`channels unwatch`), and `DIALOGS:` the synced chats missing from your dialog list
152+
that Telegram says you are still in. With `--json`: `leftChats`.
150153

151154
## Update notice
152155
Once a day tgcli asks the npm registry for the latest `@kfastov/tgcli` and keeps the
@@ -189,6 +192,12 @@ Update available: tgcli 2.2.5 -> 2.3.0. Run `brew update && brew upgrade tgcli`
189192
- Subscribe a chat for archiving: enables sync and queues a backfill job; run `tgcli backfill --once` or `tgcli backfill --follow` to process it.
190193
- channels unwatch --chat <id|username>
191194
- Unsubscribe a chat from archiving (disables sync).
195+
- The server does this by itself for a channel or group you left: Telegram sends
196+
nothing from it, so the archive could not keep up. A synced chat missing from the
197+
dialog list is only a reason to ask Telegram, once a day; sync goes off when Telegram
198+
says you left or were removed, or the chat is gone, and comes back on, with what it
199+
missed, when the chat returns to your dialog list. Chats you turned off yourself stay
200+
off, and private chats are never asked about.
192201
- `channels sync --chat <id|username> --enable|--disable` remains as a hidden alias of watch/unwatch.
193202

194203
## topics (forum supergroups)

‎message-sync-service.js‎

Lines changed: 95 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import path from 'path';
33
import fs from 'fs';
44
import { setTimeout as delay } from 'timers/promises';
55
import { Message, MtPeerNotFoundError, PeersIndex, _messageMediaFromTl } from '@mtcute/core';
6-
import { normalizeChannelId, summarizeMedia } from './telegram-client.js';
6+
import { isChatGone, normalizeChannelId, summarizeMedia } from './telegram-client.js';
77
import { resolveStoreDir, resolveStorePaths } from './core/store.js';
88
import { parseRequiredWaitSeconds } from './core/send-utils.js';
99

@@ -32,6 +32,8 @@ const RECHECK_RECENT_MS = 10 * 60 * 1000;
3232
// A read that keeps failing for this long, on a running server, is not going to work.
3333
const RECHECK_GIVE_UP_MS = 24 * 60 * 60 * 1000;
3434
const PERIODIC_RECONCILE_MS = 60 * 60 * 1000;
35+
// A chat missing from the dialog list but still joined is asked about again this often.
36+
const MEMBERSHIP_RECHECK_MS = 24 * 60 * 60 * 1000;
3537
const CONNECTION_CHECK_TIMEOUT_MS = 20 * 1000;
3638
const RETRY_BASE_MS = 60 * 1000;
3739
// A repair walks history one page per queue step, so jobs and gap fills never wait
@@ -227,22 +229,6 @@ function hasSequentialIds(channelId) {
227229

228230
// Telegram refusing this chat will not change soon, unlike a dropped connection, an
229231
// expired session (401) or a failure of its own (5xx), which say nothing about the message.
230-
// Errors about the chat itself, the ones a repair can give up on: the account has left it,
231-
// it is gone, or Telegram does not know it. Anything else, such as a broken session,
232-
// would stop every chat in a row and must stay a failure that is retried.
233-
const CHAT_GONE_ERRORS = new Set([
234-
'CHANNEL_INVALID',
235-
'CHANNEL_PRIVATE',
236-
'CHAT_FORBIDDEN',
237-
'CHAT_ID_INVALID',
238-
'PEER_ID_INVALID',
239-
'USER_BANNED_IN_CHANNEL',
240-
]);
241-
242-
function isChatGone(error) {
243-
return error instanceof MtPeerNotFoundError || CHAT_GONE_ERRORS.has(error?.text);
244-
}
245-
246232
function isRefusedRead(error) {
247233
const code = error?.code;
248234
return (code >= 400 && code < 500 && code !== 401) || error instanceof MtPeerNotFoundError;
@@ -659,6 +645,9 @@ export default class MessageSyncService {
659645
gap_after_message_id INTEGER,
660646
newer_sync_pending INTEGER NOT NULL DEFAULT 0,
661647
in_dialogs INTEGER,
648+
sync_off_reason TEXT,
649+
membership TEXT,
650+
membership_checked_at INTEGER,
662651
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
663652
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
664653
);
@@ -716,6 +705,12 @@ export default class MessageSyncService {
716705
this._ensureChannelColumn('gap_after_message_id', 'INTEGER');
717706
// 1 when the last full dialog list had the chat, 0 when it did not, NULL before any.
718707
this._ensureChannelColumn('in_dialogs', 'INTEGER');
708+
// 'left' when the server turned sync off because Telegram said the account left.
709+
this._ensureChannelColumn('sync_off_reason', 'TEXT');
710+
// What Telegram last said about a chat missing from the dialog list ('member', 'left',
711+
// 'gone' or 'unknown'), and when (ms).
712+
this._ensureChannelColumn('membership', 'TEXT');
713+
this._ensureChannelColumn('membership_checked_at', 'INTEGER');
719714
if (this._ensureChannelColumn('newer_sync_pending', 'INTEGER NOT NULL DEFAULT 0')) {
720715
// Jobs queued before this column existed still have their newer sync ahead of them.
721716
this.db.prepare(`
@@ -1390,10 +1385,74 @@ export default class MessageSyncService {
13901385
const dialogs = await this.telegramClient.listDialogs(0, { includeArchived: true });
13911386
this.upsertChannels(dialogs);
13921387
this._markDialogMembership(dialogs);
1388+
this._restoreReturnedChats();
13931389
this._markGapsBehindDialogs(dialogs);
13941390
return dialogs.length;
13951391
}
13961392

1393+
// A chat whose sync the server turned off when the account left is back in the dialog
1394+
// list: the account rejoined. Sync comes back, and the gap behind the cursor is filled.
1395+
_restoreReturnedChats() {
1396+
const returned = this.db.prepare(`
1397+
UPDATE channels
1398+
SET sync_enabled = 1, sync_off_reason = NULL, membership = NULL, membership_checked_at = NULL,
1399+
updated_at = CURRENT_TIMESTAMP
1400+
WHERE sync_off_reason = 'left' AND in_dialogs = 1
1401+
RETURNING channel_id
1402+
`).all().map((row) => row.channel_id);
1403+
if (returned.length) {
1404+
console.log(`[sync] ${returned.length} chat(s) the account had left are back in the dialog list, sync is on again: ${returned.join(', ')}.`);
1405+
}
1406+
}
1407+
1408+
// Telegram sends nothing from a channel or group the account has left, so the archive
1409+
// cannot keep up with it. Missing from the dialog list is only a reason to ask: sync goes
1410+
// off only when Telegram says the account left or was removed, or the chat is gone. One
1411+
// still joined keeps its sync and is asked about again a day later. Private chats are not
1412+
// asked about: their messages arrive without a dialog.
1413+
async checkChatsMissingFromDialogs() {
1414+
const dueBefore = Date.now() - MEMBERSHIP_RECHECK_MS;
1415+
const missing = this.db.prepare(`
1416+
SELECT channel_id FROM channels
1417+
WHERE sync_enabled = 1 AND in_dialogs = 0 AND channel_id GLOB '-[0-9]*'
1418+
AND (membership_checked_at IS NULL OR membership_checked_at < ?)
1419+
ORDER BY channel_id
1420+
`).all(dueBefore).map((row) => row.channel_id);
1421+
if (!missing.length) {
1422+
return null;
1423+
}
1424+
const membership = await this.telegramClient.getChatMembership(missing);
1425+
const counts = { off: 0, member: 0, unknown: 0 };
1426+
const now = Date.now();
1427+
const turnOff = this.db.prepare(`
1428+
UPDATE channels
1429+
SET sync_enabled = 0, sync_off_reason = 'left', membership = ?, membership_checked_at = ?,
1430+
updated_at = CURRENT_TIMESTAMP
1431+
WHERE channel_id = ? AND sync_enabled = 1 AND in_dialogs = 0
1432+
`);
1433+
const checked = this.db.prepare('UPDATE channels SET membership = ?, membership_checked_at = ? WHERE channel_id = ?');
1434+
const stillJoined = [];
1435+
this.db.transaction(() => {
1436+
for (const channelId of missing) {
1437+
const state = membership.get(channelId) ?? 'unknown';
1438+
if (state === 'left' || state === 'gone') {
1439+
counts.off += turnOff.run(state, now, channelId).changes;
1440+
} else {
1441+
checked.run(state, now, channelId);
1442+
counts[state === 'member' ? 'member' : 'unknown'] += 1;
1443+
if (state === 'member') {
1444+
stillJoined.push(channelId);
1445+
}
1446+
}
1447+
}
1448+
})();
1449+
console.log(`[sync] ${missing.length} synced chat(s) are missing from the dialog list: sync off for ${counts.off} the account left, ${counts.member} still joined, ${counts.unknown} unknown.`);
1450+
if (stillJoined.length) {
1451+
console.warn(`[sync] Missing from the dialog list but still joined, sync stays on: ${stillJoined.join(', ')}.`);
1452+
}
1453+
return counts;
1454+
}
1455+
13971456
// A chat missing from the full dialog list is one the account left or lost: Telegram
13981457
// sends no updates for it, and the archive can no longer keep up with it. An empty list
13991458
// proves nothing, so it changes nothing.
@@ -1443,6 +1502,12 @@ export default class MessageSyncService {
14431502
} while (this.reconcileAgain && !this.stopRequested);
14441503
this.reconcileNeeded = false;
14451504
this.lastReconcileAt = Date.now();
1505+
if (!this.stopRequested) {
1506+
// After the comparison, so its requests do not hold the cursors any longer.
1507+
void this.checkChatsMissingFromDialogs().catch((error) => {
1508+
console.error(`[sync] Could not ask Telegram about chats missing from the dialog list: ${error?.message ?? error}`);
1509+
});
1510+
}
14461511
return this.stopRequested ? null : dialogCount;
14471512
} finally {
14481513
// In the same tick as the last look at reconcileAgain, so no request falls between.
@@ -1567,6 +1632,9 @@ export default class MessageSyncService {
15671632
VALUES (?, ?, CURRENT_TIMESTAMP)
15681633
ON CONFLICT(channel_id) DO UPDATE SET
15691634
sync_enabled = excluded.sync_enabled,
1635+
sync_off_reason = NULL,
1636+
membership = NULL,
1637+
membership_checked_at = NULL,
15701638
updated_at = CURRENT_TIMESTAMP
15711639
RETURNING channel_id, sync_enabled;
15721640
`);
@@ -2683,6 +2751,16 @@ export default class MessageSyncService {
26832751
// Archives written before 2.2.5 can hold holes nothing fills by itself. Finding them
26842752
// costs as much as filling them, so this only says whether `--all` has run, or the
26852753
// archive was started by a version with the fixes.
2754+
// What the server did about synced chats missing from the dialog list.
2755+
getLeftChatCounts() {
2756+
return this.db.prepare(`
2757+
SELECT
2758+
COALESCE(SUM(sync_off_reason = 'left' AND sync_enabled = 0), 0) AS syncOff,
2759+
COALESCE(SUM(sync_enabled = 1 AND in_dialogs = 0 AND membership = 'member'), 0) AS stillJoined
2760+
FROM channels
2761+
`).get();
2762+
}
2763+
26862764
isRepairSuggested() {
26872765
if (this.db.prepare("SELECT 1 FROM search_meta WHERE key = 'archive_repair'").get()) {
26882766
return false;

‎telegram-client.js‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { TelegramClient as MtCuteClient } from '@mtcute/node';
2-
import { InputMedia, Message, PeersIndex } from '@mtcute/core';
2+
import { InputMedia, Message, MtPeerNotFoundError, PeersIndex } from '@mtcute/core';
33
import { randomLong, toggleChannelIdMark } from '@mtcute/core/utils.js';
44
import { html } from '@mtcute/html-parser';
55
import { md } from '@mtcute/markdown-parser';
@@ -508,6 +508,43 @@ function resolveDownloadLocation(media) {
508508
return null;
509509
}
510510

511+
// Errors about a chat itself: the account has left it or was removed, it is gone, or
512+
// Telegram does not know it. Anything else, such as a broken session, says nothing
513+
// about the chat.
514+
const CHAT_GONE_ERRORS = new Set([
515+
'CHANNEL_INVALID',
516+
'CHANNEL_PRIVATE',
517+
'CHAT_FORBIDDEN',
518+
'CHAT_ID_INVALID',
519+
'PEER_ID_INVALID',
520+
'USER_BANNED_IN_CHANNEL',
521+
]);
522+
523+
export function isChatGone(error) {
524+
return error instanceof MtPeerNotFoundError || CHAT_GONE_ERRORS.has(error?.text);
525+
}
526+
527+
// What Telegram says about the account in a channel or group: 'member', 'left' (left on
528+
// its own), 'gone' (removed, banned, the chat deleted or migrated) or 'unknown' (not in
529+
// the local peer cache, so there is nothing to ask with).
530+
function membershipOf(chat) {
531+
switch (chat?._) {
532+
case 'channel':
533+
return chat.left ? 'left' : 'member';
534+
case 'chat':
535+
if (chat.deactivated || chat.migratedTo) {
536+
return 'gone';
537+
}
538+
return chat.left ? 'left' : 'member';
539+
case 'channelForbidden':
540+
case 'chatForbidden':
541+
case 'chatEmpty':
542+
return 'gone';
543+
default:
544+
return null;
545+
}
546+
}
547+
511548
export function normalizeChannelId(channelId) {
512549
if (typeof channelId === 'number') {
513550
return channelId;
@@ -855,6 +892,63 @@ class TelegramClient {
855892
return true;
856893
}
857894

895+
// Asks Telegram about the account's membership in channels and groups, given marked
896+
// ids, with one request per 100 of each kind. Returns a Map of id to 'member', 'left',
897+
// 'gone' or 'unknown'; private chats have no membership and are left out.
898+
async getChatMembership(chatIds) {
899+
await this.ensureLogin();
900+
const result = new Map();
901+
const channels = [];
902+
const groups = [];
903+
for (const id of chatIds) {
904+
let peer;
905+
try {
906+
peer = await this.client.resolvePeer(Number(id));
907+
} catch (error) {
908+
if (!(error instanceof MtPeerNotFoundError)) {
909+
throw error;
910+
}
911+
result.set(String(id), 'unknown');
912+
continue;
913+
}
914+
if (peer._ === 'inputPeerChannel') {
915+
channels.push({ id: String(id), input: { _: 'inputChannel', channelId: peer.channelId, accessHash: peer.accessHash } });
916+
} else if (peer._ === 'inputPeerChat') {
917+
groups.push({ id: String(id), input: peer.chatId });
918+
}
919+
}
920+
921+
const ask = async (items, request, markedId) => {
922+
for (let start = 0; start < items.length; start += 100) {
923+
const batch = items.slice(start, start + 100);
924+
let chats;
925+
try {
926+
({ chats } = await this.client.call(request(batch.map((item) => item.input))));
927+
} catch (error) {
928+
if (!isChatGone(error)) {
929+
throw error;
930+
}
931+
if (batch.length === 1) {
932+
result.set(batch[0].id, 'gone');
933+
continue;
934+
}
935+
// One refused chat fails the whole batch, so ask about each on its own.
936+
for (const item of batch) {
937+
await ask([item], request, markedId);
938+
}
939+
continue;
940+
}
941+
const byId = new Map(chats.map((chat) => [markedId(chat.id), membershipOf(chat)]));
942+
for (const item of batch) {
943+
result.set(item.id, byId.get(item.id) ?? 'unknown');
944+
}
945+
}
946+
};
947+
await ask(channels, (id) => ({ _: 'channels.getChannels', id }), (rawId) => String(toggleChannelIdMark(rawId)));
948+
await ask(groups, (id) => ({ _: 'messages.getChats', id }), (rawId) => String(-rawId));
949+
return result;
950+
}
951+
858952
async listDialogs(limit = 50, { includeArchived = false, retriedAfterSessionReset = false } = {}) {
859953
try {
860954
await this.ensureLogin();

‎tests/doctor-update.test.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ vi.mock('../core/services.js', () => ({
2323
getSearchStatus: () => ({ enabled: true, version: 5 }),
2424
getQueueStats: () => ({ pending: 0, in_progress: 0, idle: 0, error: 0, gaps: 0 }),
2525
isRepairSuggested: () => false,
26+
getLeftChatCounts: () => ({ syncOff: 0, stillJoined: 0 }),
2627
close: vi.fn().mockResolvedValue(undefined),
2728
},
2829
})),

0 commit comments

Comments
 (0)