From 9c2685d460142e76bc626819db0dbeb3ecc390bc Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 24 Jul 2026 14:53:42 +0100 Subject: [PATCH] add support for reviews --- .gitignore | 1 + DEV.md | 11 +- fastanki/__init__.py | 1 + fastanki/_modidx.py | 118 ++ fastanki/collection.py | 65 +- fastanki/core.py | 51 +- fastanki/fsrs.py | 108 ++ fastanki/scheduler.py | 566 ++++++++++ nbs/01_collection.ipynb | 220 +++- nbs/03_fsrs.ipynb | 374 +++++++ nbs/04_scheduler.ipynb | 1532 ++++++++++++++++++++++++++ nbs/{03_core.ipynb => 05_core.ipynb} | 325 +++++- nbs/index.ipynb | 235 +++- tests/test_sync.py | 44 +- 14 files changed, 3560 insertions(+), 91 deletions(-) create mode 100644 fastanki/fsrs.py create mode 100644 fastanki/scheduler.py create mode 100644 nbs/03_fsrs.ipynb create mode 100644 nbs/04_scheduler.ipynb rename nbs/{03_core.ipynb => 05_core.ipynb} (58%) diff --git a/.gitignore b/.gitignore index 8615ef1..c0652d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ anki/ +fsrs-rs/ _docs/ _proc/ diff --git a/DEV.md b/DEV.md index 932af73..5b0e332 100644 --- a/DEV.md +++ b/DEV.md @@ -9,9 +9,11 @@ This is an nbdev project. The notebooks in `nbs/` are the source of truth, and ` - `00_schema.ipynb` (`schema.py`): the sqlite file format. Schema 18 DDL, the `unicase` collation, timestamps and id allocation, the default deck and deck config, the Basic and Cloze notetypes, and `create_collection`. - `01_collection.ipynb` (`collection.py`): the `Collection` class. Notes, cards, decks, card generation, search, due counts, and the checksum, guid, and HTML stripping utilities Anki uses to fingerprint notes. - `02_syncer.ipynb` (`syncer.py`): the AnkiWeb client. Login, the delta sync state machine, schema 11 conversions, full upload and download, and auth storage. -- `03_core.ipynb` (`core.py`): the functional API (`add_card`, `find_notes`, `sync`, etc). Each function opens the default collection, does its work, and closes. `anki_tools` exposes these as LLM tools. +- `03_fsrs.ipynb` (`fsrs.py`): the FSRS-6 memory model, a pure port of the scheduling half of the `fsrs-rs` crate Anki embeds: parameter upgrading/clipping (17/19/21-length sets), the forgetting curve, memory-state updates, and the SM-2 approximation for truncated histories. No card or collection knowledge. +- `04_scheduler.ipynb` (`scheduler.py`): reviewing. The v3 scheduler state machine (new/learning/review/relearning, SM-2 and FSRS), interval fuzz, `answer_card` with its revlog/counter/leech bookkeeping, `answer_buttons`, and a simplified `next_card` queue with daily limits and sibling burying. +- `05_core.ipynb` (`core.py`): the functional API (`add_card`, `find_notes`, `sync`, etc). Each function opens the default collection, does its work, and closes. `anki_tools` exposes these as LLM tools. -`fastanki/_proto/` is not generated from a notebook. It contains protobuf modules copied from the `anki` wheel with imports rewritten, and can be refreshed the same way from a newer wheel if needed. `tests/test_sync.py` is the end to end sync test, described below. `anki/` (gitignored) is a clone of the Anki source, kept for reference. When a protocol question comes up, the answer is in `anki/rslib/src/sync/`. +`fastanki/_proto/` is not generated from a notebook. It contains protobuf modules copied from the `anki` wheel with imports rewritten, and can be refreshed the same way from a newer wheel if needed. `tests/test_sync.py` is the end to end sync test, described below. `anki/` (gitignored) is a clone of the Anki source, kept for reference. When a protocol question comes up, the answer is in `anki/rslib/src/sync/`; for scheduling, `anki/rslib/src/scheduler/` and the `fsrs-rs/` clone (also gitignored). ## Design @@ -50,6 +52,9 @@ Hard-won facts worth keeping. When a protocol question comes up the answer is in - `config.curModel` is a per-collection notetype id (a timestamp), so exclude it when comparing the `config` table against the oracle. - The AnkiWeb sync client lives in `syncer.py` (module `fastanki.syncer`), named so it doesn't collide with core's top-level `sync` function; `from fastanki import *` exposes both, and `SyncServer`/`sync_collection`/etc. come from `fastanki.syncer`. - `apsw` (not stdlib `sqlite3`): connections have no `.commit()`/`.rollback()` (autocommit unless inside an explicit transaction), `execute` runs multiple statements so there is no `executescript`, and double-quoted strings are disabled, so SQL string literals must use single quotes. Mutations go through `Collection._tx()`; sync uses explicit `BEGIN IMMEDIATE`/`COMMIT`/`ROLLBACK`. The busy timeout is set before the WAL pragma, else concurrent opens hit `BusyError` on the journal-mode switch. +- Scheduling gotchas, learned porting `rslib/src/scheduler/` for `04_scheduler.ipynb`: `ANKI_TEST_MODE=1` (set before the `anki` package first computes states) disables the oracle's interval fuzz, which is what makes exact scheduling comparisons possible. Fuzz is seeded per `(card_id + reps)`, but Anki seeds Rust's ChaCha12 and we seed Python's Mersenne twister, so the two pick different (equally valid) points inside identical fuzz bounds; everything else is deterministic. Anki computes in f32: keep the protobuf floats raw (1.3 is really 1.2999999523) and round half-away-from-zero (`_round`), or intervals drift by a day at rounding boundaries. +- fsrs-rs version matters: Anki 26.05 pins fsrs 6.6.1, whose same-day (short-term) stability floors the multiplier at 1 for Good/Easy only; 6.6.2+ floors Hard too. We match 6.6.1 -- the twin-collection oracle test caught this, and it's the arbiter if the pin moves. +- The `cards.data` JSON carries `pos` (original new-queue position), `lrt` (last review time), and under FSRS `s`/`d`/`dr`/`decay`, rounded to 4/3/2/3 decimal places respectively. Anki clears `s`/`d`/`dr` when answering with FSRS off but leaves `decay` alone. ## Not implemented @@ -57,4 +62,4 @@ Media sync is a separate protocol and is skipped entirely. That is safe, because Filtered decks aren't supported. The schema 11 conversions assert if they meet one, so syncing such a collection fails rather than corrupting it. -There is no scheduler. We count due cards but can't answer them. If reviewing is ever wanted, FSRS is available as a pure python package, so the Rust scheduler wouldn't need porting. +The scheduler covers normal decks only: answering a card in a filtered deck raises. The `next_card` queue is a simplified port of Anki's v3 builder -- daily limits come from the named deck's preset rather than a per-subdeck limit tree, gathering is by due order, reviews precede new cards, and there is no display-order matrix. Those simplifications affect session ordering, never scheduling state, so they can be extended without compatibility concerns. The FSRS optimizer is deliberately absent: parameters arrive through deck-config sync, and users can optimize from Anki desktop (scheduling with synced-or-default parameters is exactly what AnkiWeb's own study feature does). diff --git a/fastanki/__init__.py b/fastanki/__init__.py index bb6ac95..f2caf53 100644 --- a/fastanki/__init__.py +++ b/fastanki/__init__.py @@ -1,4 +1,5 @@ __version__ = "0.0.4" from .collection import * from .syncer import * +from .scheduler import * from .core import * diff --git a/fastanki/_modidx.py b/fastanki/_modidx.py index 9c6aba8..ff6cd7f 100644 --- a/fastanki/_modidx.py +++ b/fastanki/_modidx.py @@ -32,6 +32,7 @@ 'fastanki.collection.Collection._tx': ('collection.html#collection._tx', 'fastanki/collection.py'), 'fastanki.collection.Collection.add': ('collection.html#collection.add', 'fastanki/collection.py'), 'fastanki.collection.Collection.close': ('collection.html#collection.close', 'fastanki/collection.py'), + 'fastanki.collection.Collection.conf': ('collection.html#collection.conf', 'fastanki/collection.py'), 'fastanki.collection.Collection.deck_id': ( 'collection.html#collection.deck_id', 'fastanki/collection.py'), 'fastanki.collection.Collection.decks': ('collection.html#collection.decks', 'fastanki/collection.py'), @@ -57,6 +58,8 @@ 'fastanki/collection.py'), 'fastanki.collection.Collection.remove_notes': ( 'collection.html#collection.remove_notes', 'fastanki/collection.py'), + 'fastanki.collection.Collection.timing': ( 'collection.html#collection.timing', + 'fastanki/collection.py'), 'fastanki.collection.Collection.today': ('collection.html#collection.today', 'fastanki/collection.py'), 'fastanki.collection.Collection.update_note': ( 'collection.html#collection.update_note', 'fastanki/collection.py'), @@ -81,21 +84,136 @@ 'fastanki.collection.note_cards': ('collection.html#note_cards', 'fastanki/collection.py'), 'fastanki.collection.renders_with_fields': ( 'collection.html#renders_with_fields', 'fastanki/collection.py'), + 'fastanki.collection.sched_timing': ('collection.html#sched_timing', 'fastanki/collection.py'), 'fastanki.collection.strip_html_media': ( 'collection.html#strip_html_media', 'fastanki/collection.py')}, 'fastanki.core': { 'fastanki.core.add_card': ('core.html#add_card', 'fastanki/core.py'), 'fastanki.core.add_cloze_card': ('core.html#add_cloze_card', 'fastanki/core.py'), 'fastanki.core.add_fb_card': ('core.html#add_fb_card', 'fastanki/core.py'), 'fastanki.core.anki_tools': ('core.html#anki_tools', 'fastanki/core.py'), + 'fastanki.core.answer_buttons': ('core.html#answer_buttons', 'fastanki/core.py'), + 'fastanki.core.answer_card': ('core.html#answer_card', 'fastanki/core.py'), 'fastanki.core.del_note': ('core.html#del_note', 'fastanki/core.py'), + 'fastanki.core.due_counts': ('core.html#due_counts', 'fastanki/core.py'), 'fastanki.core.find_card_ids': ('core.html#find_card_ids', 'fastanki/core.py'), 'fastanki.core.find_cards': ('core.html#find_cards', 'fastanki/core.py'), 'fastanki.core.find_note_ids': ('core.html#find_note_ids', 'fastanki/core.py'), 'fastanki.core.find_notes': ('core.html#find_notes', 'fastanki/core.py'), 'fastanki.core.get_note': ('core.html#get_note', 'fastanki/core.py'), + 'fastanki.core.next_card': ('core.html#next_card', 'fastanki/core.py'), 'fastanki.core.sync': ('core.html#sync', 'fastanki/core.py'), 'fastanki.core.update_fb_note': ('core.html#update_fb_note', 'fastanki/core.py'), 'fastanki.core.update_note': ('core.html#update_note', 'fastanki/core.py')}, + 'fastanki.fsrs': { 'fastanki.fsrs.MemSt': ('fsrs.html#memst', 'fastanki/fsrs.py'), + 'fastanki.fsrs._init_d': ('fsrs.html#_init_d', 'fastanki/fsrs.py'), + 'fastanki.fsrs._next_d': ('fsrs.html#_next_d', 'fastanki/fsrs.py'), + 'fastanki.fsrs._s_fail': ('fsrs.html#_s_fail', 'fastanki/fsrs.py'), + 'fastanki.fsrs._s_short_term': ('fsrs.html#_s_short_term', 'fastanki/fsrs.py'), + 'fastanki.fsrs._s_success': ('fsrs.html#_s_success', 'fastanki/fsrs.py'), + 'fastanki.fsrs.clamp': ('fsrs.html#clamp', 'fastanki/fsrs.py'), + 'fastanki.fsrs.f32': ('fsrs.html#f32', 'fastanki/fsrs.py'), + 'fastanki.fsrs.forgetting_curve': ('fsrs.html#forgetting_curve', 'fastanki/fsrs.py'), + 'fastanki.fsrs.fsrs_params': ('fsrs.html#fsrs_params', 'fastanki/fsrs.py'), + 'fastanki.fsrs.memory_state_from_sm2': ('fsrs.html#memory_state_from_sm2', 'fastanki/fsrs.py'), + 'fastanki.fsrs.next_interval': ('fsrs.html#next_interval', 'fastanki/fsrs.py'), + 'fastanki.fsrs.next_states': ('fsrs.html#next_states', 'fastanki/fsrs.py'), + 'fastanki.fsrs.param_decay': ('fsrs.html#param_decay', 'fastanki/fsrs.py'), + 'fastanki.fsrs.step': ('fsrs.html#step', 'fastanki/fsrs.py')}, + 'fastanki.scheduler': { 'fastanki.scheduler.Answers': ('scheduler.html#answers', 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection._bump_deck_stats': ( 'scheduler.html#collection._bump_deck_stats', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection._bury_siblings': ( 'scheduler.html#collection._bury_siblings', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection._card_answers': ( 'scheduler.html#collection._card_answers', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection._day_limits': ( 'scheduler.html#collection._day_limits', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection._deck_dr': ( 'scheduler.html#collection._deck_dr', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection._fsrs_ctx': ( 'scheduler.html#collection._fsrs_ctx', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection._leech_note': ( 'scheduler.html#collection._leech_note', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection.answer_buttons': ( 'scheduler.html#collection.answer_buttons', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection.answer_card': ( 'scheduler.html#collection.answer_card', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection.deck_conf': ( 'scheduler.html#collection.deck_conf', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection.fsrs_on': ('scheduler.html#collection.fsrs_on', 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection.next_card': ( 'scheduler.html#collection.next_card', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Collection.unbury_if_day_changed': ( 'scheduler.html#collection.unbury_if_day_changed', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.Ctx': ('scheduler.html#ctx', 'fastanki/scheduler.py'), + 'fastanki.scheduler.LearnSt': ('scheduler.html#learnst', 'fastanki/scheduler.py'), + 'fastanki.scheduler.LearnSt.ivl_kind': ('scheduler.html#learnst.ivl_kind', 'fastanki/scheduler.py'), + 'fastanki.scheduler.LearnSt.next_answers': ( 'scheduler.html#learnst.next_answers', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.LearnSt.revlog_kind': ( 'scheduler.html#learnst.revlog_kind', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.NewSt': ('scheduler.html#newst', 'fastanki/scheduler.py'), + 'fastanki.scheduler.NewSt.ivl_kind': ('scheduler.html#newst.ivl_kind', 'fastanki/scheduler.py'), + 'fastanki.scheduler.NewSt.next_answers': ('scheduler.html#newst.next_answers', 'fastanki/scheduler.py'), + 'fastanki.scheduler.NewSt.revlog_kind': ('scheduler.html#newst.revlog_kind', 'fastanki/scheduler.py'), + 'fastanki.scheduler.RelearnSt': ('scheduler.html#relearnst', 'fastanki/scheduler.py'), + 'fastanki.scheduler.RelearnSt.ivl_kind': ('scheduler.html#relearnst.ivl_kind', 'fastanki/scheduler.py'), + 'fastanki.scheduler.RelearnSt.next_answers': ( 'scheduler.html#relearnst.next_answers', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.RelearnSt.revlog_kind': ( 'scheduler.html#relearnst.revlog_kind', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.ReviewSt': ('scheduler.html#reviewst', 'fastanki/scheduler.py'), + 'fastanki.scheduler.ReviewSt._failing_ivl': ( 'scheduler.html#reviewst._failing_ivl', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.ReviewSt._passing_early_ivls': ( 'scheduler.html#reviewst._passing_early_ivls', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.ReviewSt._passing_fsrs_ivls': ( 'scheduler.html#reviewst._passing_fsrs_ivls', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.ReviewSt._passing_ivls': ( 'scheduler.html#reviewst._passing_ivls', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.ReviewSt.ivl_kind': ('scheduler.html#reviewst.ivl_kind', 'fastanki/scheduler.py'), + 'fastanki.scheduler.ReviewSt.next_answers': ( 'scheduler.html#reviewst.next_answers', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.ReviewSt.revlog_kind': ( 'scheduler.html#reviewst.revlog_kind', + 'fastanki/scheduler.py'), + 'fastanki.scheduler._card_data': ('scheduler.html#_card_data', 'fastanki/scheduler.py'), + 'fastanki.scheduler._constrain_passing': ('scheduler.html#_constrain_passing', 'fastanki/scheduler.py'), + 'fastanki.scheduler._fsrs_reviews': ('scheduler.html#_fsrs_reviews', 'fastanki/scheduler.py'), + 'fastanki.scheduler._graduate': ('scheduler.html#_graduate', 'fastanki/scheduler.py'), + 'fastanki.scheduler._learn_short': ('scheduler.html#_learn_short', 'fastanki/scheduler.py'), + 'fastanki.scheduler._mem': ('scheduler.html#_mem', 'fastanki/scheduler.py'), + 'fastanki.scheduler._memory_from_revlog': ( 'scheduler.html#_memory_from_revlog', + 'fastanki/scheduler.py'), + 'fastanki.scheduler._min_max': ('scheduler.html#_min_max', 'fastanki/scheduler.py'), + 'fastanki.scheduler._relearn_pass': ('scheduler.html#_relearn_pass', 'fastanki/scheduler.py'), + 'fastanki.scheduler._rl_days': ('scheduler.html#_rl_days', 'fastanki/scheduler.py'), + 'fastanki.scheduler._round': ('scheduler.html#_round', 'fastanki/scheduler.py'), + 'fastanki.scheduler._round_days': ('scheduler.html#_round_days', 'fastanki/scheduler.py'), + 'fastanki.scheduler._shifted_d': ('scheduler.html#_shifted_d', 'fastanki/scheduler.py'), + 'fastanki.scheduler._step_idx': ('scheduler.html#_step_idx', 'fastanki/scheduler.py'), + 'fastanki.scheduler._step_secs': ('scheduler.html#_step_secs', 'fastanki/scheduler.py'), + 'fastanki.scheduler.allow_short_term': ('scheduler.html#allow_short_term', 'fastanki/scheduler.py'), + 'fastanki.scheduler.card_state': ('scheduler.html#card_state', 'fastanki/scheduler.py'), + 'fastanki.scheduler.fuzz_bounds': ('scheduler.html#fuzz_bounds', 'fastanki/scheduler.py'), + 'fastanki.scheduler.fuzz_delta': ('scheduler.html#fuzz_delta', 'fastanki/scheduler.py'), + 'fastanki.scheduler.fuzz_factor': ('scheduler.html#fuzz_factor', 'fastanki/scheduler.py'), + 'fastanki.scheduler.ignore_revlogs_before': ( 'scheduler.html#ignore_revlogs_before', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.ivl_days': ('scheduler.html#ivl_days', 'fastanki/scheduler.py'), + 'fastanki.scheduler.ivl_secs': ('scheduler.html#ivl_secs', 'fastanki/scheduler.py'), + 'fastanki.scheduler.learn_fuzz': ('scheduler.html#learn_fuzz', 'fastanki/scheduler.py'), + 'fastanki.scheduler.leech_threshold_met': ( 'scheduler.html#leech_threshold_met', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.min_fuzz_ivl': ('scheduler.html#min_fuzz_ivl', 'fastanki/scheduler.py'), + 'fastanki.scheduler.mk_ctx': ('scheduler.html#mk_ctx', 'fastanki/scheduler.py'), + 'fastanki.scheduler.preset_params': ('scheduler.html#preset_params', 'fastanki/scheduler.py'), + 'fastanki.scheduler.step_again_delay': ('scheduler.html#step_again_delay', 'fastanki/scheduler.py'), + 'fastanki.scheduler.step_current_delay': ('scheduler.html#step_current_delay', 'fastanki/scheduler.py'), + 'fastanki.scheduler.step_good_delay': ('scheduler.html#step_good_delay', 'fastanki/scheduler.py'), + 'fastanki.scheduler.step_hard_delay': ('scheduler.html#step_hard_delay', 'fastanki/scheduler.py'), + 'fastanki.scheduler.step_remaining_good': ( 'scheduler.html#step_remaining_good', + 'fastanki/scheduler.py'), + 'fastanki.scheduler.with_fuzz': ('scheduler.html#with_fuzz', 'fastanki/scheduler.py')}, 'fastanki.schema': { 'fastanki.schema.add_basic': ('schema.html#add_basic', 'fastanki/schema.py'), 'fastanki.schema.add_cloze': ('schema.html#add_cloze', 'fastanki/schema.py'), 'fastanki.schema.add_notetype': ('schema.html#add_notetype', 'fastanki/schema.py'), diff --git a/fastanki/collection.py b/fastanki/collection.py index ffb9270..c932f14 100644 --- a/fastanki/collection.py +++ b/fastanki/collection.py @@ -6,10 +6,11 @@ # %% auto #0 __all__ = ['strip_html_media', 'field_csum', 'guid64', 'data_dir', 'Collection', 'NT', 'field_is_empty', 'renders_with_fields', - 'Note', 'note_cards', 'Card'] + 'Note', 'note_cards', 'sched_timing', 'Card'] # %% ../nbs/01_collection.ipynb #04a4306c import hashlib, html, random, time, json +from datetime import datetime, timezone, timedelta from contextlib import contextmanager from fastcore.utils import * from .schema import * @@ -279,6 +280,43 @@ def remove_deck(self:Collection, name): self._dirty() return len(dids) +# %% ../nbs/01_collection.ipynb #9a3ad061 +@patch +def conf(self:Collection, key, default=None): + "Retrieve `key`'s value from the config table, or `default` if absent" + return default if (v := self.q1('select val from config where key=?', key)) is None else json.loads(v) + +def sched_timing( + crt, # Collection creation stamp (epoch secs) + now, # Current time (epoch secs) + crt_mins_west=None, # UTC offset at creation (`creationOffset` config; None = legacy collection) + now_mins_west=0, # Current UTC offset, e.g. from `day_offset` + rollover=4, # Hour of day when the next day starts +): + "Anki's `(days_elapsed, next_day_at)`: the scheduler day counter and next rollover time" + ndt = datetime.fromtimestamp(now, timezone(timedelta(minutes=-now_mins_west))) + roll = ndt.replace(hour=rollover%24, minute=0, second=0, microsecond=0) + if crt_mins_west is None: + crt_roll = datetime.fromtimestamp(crt, ndt.tzinfo).replace(hour=rollover%24, minute=0, second=0, microsecond=0) + days = (now - int(crt_roll.timestamp()))//86400 + passed = roll < ndt # the legacy path holds the cutoff for the whole rollover second + else: + cdt = datetime.fromtimestamp(crt, timezone(timedelta(minutes=-crt_mins_west))) + passed = roll <= ndt + days = (ndt.date() - cdt.date()).days - (not passed) + return max(days,0), int(roll.timestamp()) + 86400*passed + +@patch +def timing(self:Collection): + "`(days_elapsed, next_day_at)` for this collection, resolving its config" + return sched_timing(self.q1('select crt from col'), int(time.time()), self.conf('creationOffset'), + day_offset(), min(self.conf('rollover',4),23)) + +@patch +def today(self:Collection): + "Days since collection creation, Anki's day counter" + return self.timing()[0] + # %% ../nbs/01_collection.ipynb #f9bf6da1 @patch def _find_sql(self:Collection, deck=None, tag=None, added_days=None, where=None, args=()): @@ -312,14 +350,17 @@ def find_note_ids(self:Collection, **kw): return [n.id for n in self.find_notes( # %% ../nbs/01_collection.ipynb #2525e881 class Card: - def __init__(self, id, nid, did, ord, mod, usn, type, queue, due, ivl): store_attr() + def __init__(self, id, nid, did, ord, mod, usn, type, queue, due, ivl, factor=0, reps=0, lapses=0, left=0, odue=0, odid=0, flags=0, data=""): store_attr() def __repr__(self): return f"Card({self.id}, nid={self.nid}, due={self.due}, ivl={self.ivl}, queue={self.queue})" - def _repr_markdown_(self): return f"Card {self.id} (nid: {self.nid}, due: {self.due}, ivl: {self.ivl}d, queue: {self.queue})" - -@patch -def today(self:Collection): - "Days since collection creation, Anki's day counter" - return (int(time.time()) - self.q1('select crt from col'))//86400 + def _repr_markdown_(self): + if self.queue==0: st = f'new #{self.due}' + elif self.queue==1: + secs = max(self.due-int(time.time()), 0) + st = f"learning, due in {f'{-(-secs//3600)}h' if secs>=3600 else f'{-(-secs//60)}m'}" if secs else 'learning, due now' + elif self.queue==3: st = f'learning, due day {self.due}' + elif self.queue==2: st = f'review, ivl {self.ivl}d, due day {self.due}' + else: st = {-1:'suspended'}.get(self.queue, 'buried') + return f"Card {self.id} (nid: {self.nid}): {st}" @patch def find_cards(self:Collection, deck=None, tag=None, added_days=None, is_due=None, where=None, args=(), **fields): @@ -327,8 +368,10 @@ def find_cards(self:Collection, deck=None, tag=None, added_days=None, is_due=Non nids = None if fields: nids = {x.id for x in self.find_notes(deck=deck, tag=tag, added_days=added_days, **fields)} cond,ps = self._find_sql(deck, tag, added_days, where, args) - if is_due: cond += f' and (c.queue=1 and c.due<=? or c.queue in (2,3) and c.due<=?)'; ps += [int(time.time())+1200, self.today()] - sql = f'select c.id, c.nid, c.did, c.ord, c.mod, c.usn, c.type, c.queue, c.due, c.ivl from cards c join notes n on c.nid=n.id where {cond} order by c.id' + if is_due: + cond += f' and (c.queue=1 and c.due<=? or c.queue in (2,3) and c.due<=?)' + ps += [int(time.time())+self.conf('collapseTime',1200), self.today()] + sql = f'select c.* from cards c join notes n on c.nid=n.id where {cond} order by c.id' return [Card(*r) for r in self.q(sql, *ps) if nids is None or r[1] in nids] @patch @@ -341,5 +384,5 @@ def due_counts(self:Collection, deck=None): cond,ps = self._find_sql(deck) sql = ('select sum(c.queue=0), sum(c.queue=1 and c.due<=? or c.queue=3 and c.due<=?), sum(c.queue=2 and c.due<=?) ' f'from cards c join notes n on c.nid=n.id where {cond}') - r = self.q(sql, int(time.time())+1200, self.today(), self.today(), *ps)[0] + r = self.q(sql, int(time.time())+self.conf('collapseTime',1200), self.today(), self.today(), *ps)[0] return tuple(x or 0 for x in r) diff --git a/fastanki/core.py b/fastanki/core.py index a416235..85ad8a8 100644 --- a/fastanki/core.py +++ b/fastanki/core.py @@ -2,19 +2,21 @@ Docs: https://AnswerDotAI.github.io/fastanki/core.html.md""" -# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_core.ipynb. +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/05_core.ipynb. # %% auto #0 __all__ = ['add_card', 'add_fb_card', 'add_cloze_card', 'find_notes', 'find_note_ids', 'find_cards', 'find_card_ids', 'get_note', - 'update_note', 'update_fb_note', 'del_note', 'sync', 'anki_tools'] + 'update_note', 'update_fb_note', 'del_note', 'next_card', 'answer_buttons', 'answer_card', 'due_counts', + 'sync', 'anki_tools'] -# %% ../nbs/03_core.ipynb #3d1b9c61 +# %% ../nbs/05_core.ipynb #3d1b9c61 from fastcore.utils import * from .schema import * from .collection import * from .syncer import * +from .scheduler import * -# %% ../nbs/03_core.ipynb #eec56ce5 +# %% ../nbs/05_core.ipynb #eec56ce5 def add_card( model:str='Basic', # Notetype name (the tool description lists each notetype's fields) deck:str='Default', # Deck name (`::` for nesting; created if needed) @@ -43,7 +45,7 @@ def add_cloze_card( "Add a Cloze card (`{{c1::hidden}}` syntax), returning the new note id." return add_card(model='Cloze', deck=deck, tags=tags, fields={'Text':text, 'Back Extra':back_extra}) -# %% ../nbs/03_core.ipynb #000f044d +# %% ../nbs/05_core.ipynb #000f044d def find_notes(deck:str=None, # Deck name (matches subdecks too) tag:str=None, # Tag to match added_days:int=None, # Only notes added in the last this-many days @@ -58,7 +60,7 @@ def find_note_ids(deck:str=None, # Deck name (matches subdecks too) "Ids of notes matching all given criteria." return [n.id for n in find_notes(deck=deck, tag=tag, added_days=added_days, fields=fields)] -# %% ../nbs/03_core.ipynb #d1ddb102 +# %% ../nbs/05_core.ipynb #d1ddb102 def find_cards(deck:str=None, # Deck name (matches subdecks too) tag:str=None, # Tag to match added_days:int=None, # Only cards added in the last this-many days @@ -75,14 +77,14 @@ def find_card_ids(deck:str=None, # Deck name (matches subdecks too) "Ids of cards matching all given criteria." return [c.id for c in find_cards(deck=deck, tag=tag, added_days=added_days, is_due=is_due, fields=fields)] -# %% ../nbs/03_core.ipynb #ed586b1b +# %% ../nbs/05_core.ipynb #ed586b1b def get_note( note_id:int, # Id of the note to retrieve ): "Retrieve a note by id." with Collection.open() as col: return col.get_note(note_id) -# %% ../nbs/03_core.ipynb #c24dadf8 +# %% ../nbs/05_core.ipynb #c24dadf8 def update_note(note, tags=None, add_tags=None, **fields): "Update fields and/or tags of a `Note` or note id; `tags` replaces, `add_tags` appends." with Collection.open() as col: return col.update_note(note, tags=tags, add_tags=add_tags, **fields) @@ -108,7 +110,34 @@ def del_note( "Delete note(s) (and their cards) by `Note` or id, singly or in a list." with Collection.open() as col: return col.remove_notes(notes) -# %% ../nbs/03_core.ipynb #5f330861 +# %% ../nbs/05_core.ipynb #a3889fa2 +def next_card( + deck:str=None, # Deck name (subdecks included; default: whole collection) +): + "The next card due for study, or None when the session is done." + with Collection.open() as col: return col.next_card(deck) + +def answer_buttons( + card_id:int, # Id of the card being reviewed +): + "For each ease 1-4, `(next_state, delay_secs)`: what to show on the answer buttons." + with Collection.open() as col: return col.answer_buttons(card_id) + +def answer_card( + card_id:int, # Id of the card being reviewed + ease:int, # 1=Again 2=Hard 3=Good 4=Easy + taken_ms:int=0, # Milliseconds spent answering, for the stats +): + "Answer a due card, updating its schedule and review log." + with Collection.open() as col: return col.answer_card(card_id, ease, taken_ms=taken_ms) + +def due_counts( + deck:str=None, # Deck name (subdecks included; default: whole collection) +): + "(new, learning, review) counts due now." + with Collection.open() as col: return col.due_counts(deck) + +# %% ../nbs/05_core.ipynb #5f330861 def sync( user:str=None, # AnkiWeb email (only needed the first time) passw:str=None, # AnkiWeb password (only the first time; a host key is saved after) @@ -118,5 +147,5 @@ def sync( "Sync the default collection with AnkiWeb. Pass credentials the first time; they're saved after that." with Collection.open() as col: return col.sync(user=user, passw=passw, endpoint=endpoint, upload=upload) -# %% ../nbs/03_core.ipynb #6ab4881e -def anki_tools(): print('&`[add_card, add_fb_card, add_cloze_card, find_notes, find_note_ids, find_cards, find_card_ids, get_note, del_note, update_fb_note, sync]`') +# %% ../nbs/05_core.ipynb #6ab4881e +def anki_tools(): print('&`[add_card, add_fb_card, add_cloze_card, find_notes, find_note_ids, find_cards, find_card_ids, get_note, del_note, update_fb_note, next_card, answer_buttons, answer_card, due_counts, sync]`') diff --git a/fastanki/fsrs.py b/fastanki/fsrs.py new file mode 100644 index 0000000..a3de56b --- /dev/null +++ b/fastanki/fsrs.py @@ -0,0 +1,108 @@ +"""The FSRS-6 memory model, ported from the fsrs-rs crate that Anki embeds + +Docs: https://AnswerDotAI.github.io/fastanki/fsrs.html.md""" + +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_fsrs.ipynb. + +# %% auto #0 +__all__ = ['S_MIN', 'S_MAX', 'D_MIN', 'D_MAX', 'FSRS5_DECAY', 'FSRS6_DECAY', 'DEFAULT_PARAMS', 'ItemSt', 'f32', 'clamp', + 'fsrs_params', 'param_decay', 'forgetting_curve', 'next_interval', 'MemSt', 'step', 'next_states', + 'memory_state_from_sm2'] + +# %% ../nbs/03_fsrs.ipynb #ae880466 +import math +from struct import pack, unpack +from collections import namedtuple +from fastcore.utils import * + +# %% ../nbs/03_fsrs.ipynb #2404bf38 +S_MIN,S_MAX,D_MIN,D_MAX = 0.001,36500.0,1.0,10.0 +FSRS5_DECAY,FSRS6_DECAY = 0.5,0.1542 + +DEFAULT_PARAMS = [0.212,1.2931,2.3065,8.2956,6.4133,0.8334,3.0194,0.001,1.8722,0.1666,0.796, + 1.4835,0.0614,0.2629,1.6483,0.6014,1.8729,0.5425,0.0912,0.0658,FSRS6_DECAY] + +_CLAMPS = [(S_MIN,100.)]*4 + [(D_MIN,D_MAX),(0.001,4.0),(0.001,4.0),(0.001,0.75),(0.,4.5),(0.,0.8),(0.001,3.5), + (0.001,5.0),(0.001,0.25),(0.001,0.9),(0.,4.0),(0.,1.0),(1.0,6.0),(0.,2.0),(0.,2.0),(0.,0.8),(0.1,0.8)] + +def f32(x): return unpack('f', pack('f', x))[0] +def clamp(x, lo, hi): return min(max(x,lo),hi) + +def fsrs_params(w): + "Upgrade a 0/17/19/21-length parameter list to FSRS-6's 21 numbers and clip to legal ranges, like `FSRS::new`" + w = [float(x) for x in w] or list(DEFAULT_PARAMS) + if len(w)==17: + w[4],w[5],w[6] = w[4]+2*w[5], math.log(w[5]*3+1)/3, w[6]+0.5 + w += [0.,0.,0.,FSRS5_DECAY] + elif len(w)==19: w += [0.,FSRS5_DECAY] + assert len(w)==21, f"invalid FSRS parameter count: {len(w)}" + return [f32(clamp(x,*b)) for x,b in zip(w,_CLAMPS)] + +def param_decay(w): + "The forgetting-curve decay for a *raw* (pre-upgrade) parameter list, Anki's `get_decay_from_params`" + return FSRS6_DECAY if not len(w) else (FSRS5_DECAY if len(w)<21 else w[20]) + +# %% ../nbs/03_fsrs.ipynb #cdaadaa6 +def forgetting_curve(w, t, s): + "Probability of recall `t` days after a review that left stability `s`" + decay = -w[20] + factor = math.exp(math.log(0.9)/decay) - 1 + return f32((t/s*factor + 1)**decay) + +def next_interval(w, s, dr): + "The (fractional) days until retrievability falls to desired retention `dr`, at stability `s`" + decay = -w[20] + factor = math.exp(math.log(0.9)/decay) - 1 + return f32(s/factor*(dr**(1/decay) - 1)) + +# %% ../nbs/03_fsrs.ipynb #ced935c2 +class MemSt(namedtuple('MemSt', 'stability difficulty')): + "An FSRS memory state: `stability` in days, `difficulty` in 1-10" + +def _init_d(w, r): return w[4] - math.exp(w[5]*(r-1)) + 1 + +def _next_d(w, d, r): + nd = d + (10-d)*(-w[6]*(r-3))/9 + return clamp(w[7]*(_init_d(w,4)-nd)+nd, D_MIN, D_MAX) + +def _s_success(w, s, d, r, rating): + hp = w[15] if rating==2 else 1.0 + eb = w[16] if rating==4 else 1.0 + return s*(math.exp(w[8])*(11-d)*s**-w[9]*(math.exp((1-r)*w[10])-1)*hp*eb + 1) + +def _s_fail(w, s, d, r): + ns = w[11]*d**-w[12]*((s+1)**w[13]-1)*math.exp((1-r)*w[14]) + return min(ns, s/math.exp(w[17]*w[18])) + +def _s_short_term(w, s, rating): + "Same-day stability. Floors the multiplier at 1 for Good/Easy only" + sinc = math.exp(w[17]*(rating-3+w[18]))*s**-w[19] + return s*(max(sinc, 1.0) if rating>=3 else sinc) + +def step(w, delta_t, rating, mem): + "Memory state after rating a card `delta_t` days since its last review (`mem` None: first rating of a new card)" + if mem is None: + r = clamp(rating, 1, 4) + return MemSt(f32(clamp(w[r-1], S_MIN, S_MAX)), f32(clamp(_init_d(w,r), D_MIN, D_MAX))) + s,d = clamp(mem.stability, S_MIN, S_MAX), clamp(mem.difficulty, D_MIN, D_MAX) + r = forgetting_curve(w, delta_t, s) + if delta_t==0: ns = _s_short_term(w, s, rating) + elif rating==1: ns = _s_fail(w, s, d, r) + else: ns = _s_success(w, s, d, r, rating) + return MemSt(f32(clamp(ns, S_MIN, S_MAX)), f32(_next_d(w, d, rating))) + +ItemSt = namedtuple('ItemSt', 'mem ivl') + +def next_states(w, mem, dr, days_elapsed): + "For each rating 1-4 (index `[ease-1]`), the next `MemSt` and its desired-retention interval" + sts = [step(w, float(days_elapsed), rating, mem) for rating in (1,2,3,4)] + return [ItemSt(m, next_interval(w, m.stability, dr)) for m in sts] + +# %% ../nbs/03_fsrs.ipynb #2188716f +def memory_state_from_sm2(w, ease_factor, interval, sm2_retention=0.9): + "A `MemSt` inferred from SM-2 `ease_factor` (eg 2.5) and `interval` days" + decay = -w[20] + factor = 0.9**(1/decay) - 1 + s = max(interval, S_MIN)*factor/(sm2_retention**(1/decay) - 1) + d = 11 - (ease_factor-1)/(math.exp(w[8])*s**-w[9]*(math.exp((1-sm2_retention)*w[10]) - 1)) + return MemSt(f32(s), f32(clamp(d, D_MIN, D_MAX))) diff --git a/fastanki/scheduler.py b/fastanki/scheduler.py new file mode 100644 index 0000000..9a12e10 --- /dev/null +++ b/fastanki/scheduler.py @@ -0,0 +1,566 @@ +"""The v3 scheduler: card states, answer buttons, and the revlog, compatible with every Anki client + +Docs: https://AnswerDotAI.github.io/fastanki/scheduler.html.md""" + +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_scheduler.ipynb. + +# %% auto #0 +__all__ = ['DAY', 'EASE_AGAIN', 'EASE_HARD', 'EASE_EASY', 'MIN_EASE', 'RLog', 'NewSt', 'LearnSt', 'ReviewSt', 'RelearnSt', + 'Answers', 'ivl_days', 'ivl_secs', 'step_again_delay', 'step_hard_delay', 'step_good_delay', + 'step_current_delay', 'step_remaining_good', 'fuzz_delta', 'fuzz_bounds', 'with_fuzz', 'fuzz_factor', + 'min_fuzz_ivl', 'learn_fuzz', 'Ctx', 'mk_ctx', 'leech_threshold_met', 'card_state', 'preset_params', + 'allow_short_term', 'ignore_revlogs_before'] + +# %% ../nbs/04_scheduler.ipynb #c575cdbb +import json, time, random +from datetime import datetime, timezone +from collections import namedtuple +from fastcore.utils import * +from .schema import * +from .collection import * +from ._proto import deck_config_pb2, decks_pb2 +from .fsrs import * + +# %% ../nbs/04_scheduler.ipynb #682bf3b4 +@patch +def deck_conf(self:Collection, did): + "The parsed `DeckConfig.Config` preset governing deck `did`" + k = decks_pb2.Deck.KindContainer() + k.ParseFromString(self.q1('select kind from decks where id=?', did)) + blob = self.q1('select config from deck_config where id=?', k.normal.config_id or 1) or default_deck_config() + c = deck_config_pb2.DeckConfig.Config() + c.ParseFromString(blob) + return c + +# %% ../nbs/04_scheduler.ipynb #4fdb2d86 +class NewSt(namedtuple('NewSt', 'position')): + "Not yet studied; `position` orders the new queue" + def ivl_kind(self): return 0 + def revlog_kind(self): return 0 + +class LearnSt(namedtuple('LearnSt', 'remaining secs elapsed mem', defaults=(0,None))): + "In the learning steps: `remaining` steps left, `secs` until the next showing" + def ivl_kind(self): return -self.secs + def revlog_kind(self): return 0 + +class ReviewSt(namedtuple('ReviewSt', 'ivl ease lapses elapsed leeched mem', defaults=(0,0,False,None))): + "Graduated: due again in `ivl` days, growing by `ease`" + def ivl_kind(self): return self.ivl + def revlog_kind(self): return 3 if self.elapsed=secs_to_rollover: return (-iv-secs_to_rollover)//86400 + 1 + return iv + +def ivl_secs(iv): + "An interval in plain seconds" + return -iv if iv<0 else iv*86400 + +# %% ../nbs/04_scheduler.ipynb #b7bf1d31 +DAY = 86400 + +def _round(x): + "Round half away from zero, like Rust's `round` (Python's `round` is banker's)" + return int(x+0.5) if x>=0 else -int(-x+0.5) + +def _step_secs(steps, i): return int(steps[i]*60) if 0<=iDAY else secs + +def step_again_delay(steps): return _step_secs(steps, 0) + +def step_hard_delay(steps, remaining): + idx = _step_idx(steps, remaining) + cur = _step_secs(steps, idx) or _step_secs(steps, 0) + if cur is None: return None + if idx>0: return cur + nxt = _step_secs(steps, 1) + if nxt is not None: return _round_days((cur+nxt)//2) + return _round_days(min(cur*3//2, cur+DAY)) + +def step_good_delay(steps, remaining): return _step_secs(steps, _step_idx(steps, remaining)+1) +def step_current_delay(steps, remaining): return _step_secs(steps, _step_idx(steps, remaining)) or 0 +def step_remaining_good(steps, remaining): return len(steps)-_step_idx(steps, remaining)-1 + +# %% ../nbs/04_scheduler.ipynb #701a80dc +_FUZZ_RANGES = [(2.5,7.0,0.15), (7.0,20.0,0.1), (20.0,None,0.05)] + +def fuzz_delta(ivl): + "Days of fuzz applied either side of `ivl`" + if ivl<2.5: return 0.0 + d = 1.0 + for start,end,f in _FUZZ_RANGES: d += f*max(min(ivl, end or ivl)-start, 0) + return d + +def fuzz_bounds( + ivl, # Undisturbed interval in days (may be fractional) + lo=1, # Minimum permitted result + hi=36500, # Maximum permitted result +): + "Inclusive `(lower, upper)` day bounds for a fuzzed interval, respecting `lo`/`hi`" + lo = min(lo, hi) + ivl = clamp(ivl, lo, hi) + d = fuzz_delta(ivl) + l,u = _round(ivl-d), _round(ivl+d) + l,u = clamp(l, lo, hi), clamp(u, lo, hi) + if u==l and u>2 and u prev_ivl: return prev_ivl+1 + return prev_ivl if prev_ivl <= upper else 0 + +def learn_fuzz(fz_seed, secs): + "Intraday learning delay with Anki's up-to-25% (max 5min) extension; `fz_seed` None leaves it alone" + if fz_seed is None: return secs + upper = secs + int(min(secs*0.25, 300.0)) + if secs >= upper: return secs + return random.Random(fz_seed).randrange(secs, upper) + +# %% ../nbs/04_scheduler.ipynb #a57d10b3 +_CTX = 'fuzz steps relearn_steps grad_good grad_easy init_ease hard_mult easy_mult ivl_mult lapse_mult max_ivl min_lapse_ivl leech_threshold fsrs allow_short short_steps' +class Ctx(namedtuple('Ctx', _CTX, defaults=(None,False,False))): + "Everything a state transition needs: the deck preset's numbers plus this answer's fuzz factor" + +def mk_ctx(cfg, fuzz=None, fsrs=None, **over): + "A `Ctx` from deck preset `cfg`, overridable by keyword" + d = dict(fuzz=fuzz, steps=list(cfg.learn_steps), relearn_steps=list(cfg.relearn_steps), + grad_good=cfg.graduating_interval_good, grad_easy=cfg.graduating_interval_easy, init_ease=cfg.initial_ease, + hard_mult=cfg.hard_multiplier, easy_mult=cfg.easy_multiplier, ivl_mult=cfg.interval_multiplier, + lapse_mult=cfg.lapse_multiplier, max_ivl=cfg.maximum_review_interval, + min_lapse_ivl=cfg.minimum_lapse_interval, leech_threshold=cfg.leech_threshold, fsrs=fsrs) + d.update(over) + return Ctx(**d) + +def _min_max(ctx, minimum): + hi = max(ctx.max_ivl, 1) + return clamp(minimum, 1, hi), hi + +def leech_threshold_met(lapses, threshold): + "True at `threshold` lapses, and every half-threshold (rounded up) after" + if not threshold: return False + half = max(-(-threshold//2), 1) + return lapses>=threshold and (lapses-threshold)%half==0 + +# %% ../nbs/04_scheduler.ipynb #beff5ce2 +EASE_AGAIN,EASE_HARD,EASE_EASY,MIN_EASE = -0.2,-0.15,0.15,1.3 + +def _constrain_passing(ctx, ivl, minimum, fuzz=True): + if ctx.fsrs is None: ivl *= ctx.ivl_mult + lo,hi = _min_max(ctx, minimum) + return with_fuzz(ctx.fuzz, ivl, lo, hi) if fuzz else clamp(_round(ivl), lo, hi) + +def _mem(ctx, rating): + "The FSRS memory state this rating leads to (None under SM-2)" + return ctx.fsrs[rating-1].mem if ctx.fsrs is not None else None + +@patch +def _passing_ivls(self:ReviewSt, ctx): + "Hard/good/easy intervals, each at least a day past the one before" + if ctx.fsrs is not None: return self._passing_fsrs_ivls(ctx) + if self.elapsed < self.ivl: return self._passing_early_ivls(ctx) + cur,late = max(self.ivl,1), max(self.elapsed-self.ivl, 0) + hard_min = 0 if ctx.hard_mult<=1.0 else self.ivl+1 + hard = _constrain_passing(ctx, cur*ctx.hard_mult, hard_min) + good_min = self.ivl+1 if ctx.hard_mult<=1.0 else hard+1 + good = _constrain_passing(ctx, (cur+late/2)*self.ease, good_min) + easy = _constrain_passing(ctx, (cur+late)*self.ease*ctx.easy_mult, good+1) + return hard,good,easy + +@patch +def _passing_fsrs_ivls(self:ReviewSt, ctx): + "FSRS intervals come straight from the memory model; fuzz may not shrink an interval that grew" + ivls = [s.ivl for s in ctx.fsrs] + hard = _constrain_passing(ctx, ivls[1], max(min_fuzz_ivl(ivls[1], self.ivl, ctx.max_ivl), 1)) + good = _constrain_passing(ctx, ivls[2], max(min_fuzz_ivl(ivls[2], self.ivl, ctx.max_ivl), hard+1)) + easy = _constrain_passing(ctx, ivls[3], max(min_fuzz_ivl(ivls[3], self.ivl, ctx.max_ivl), good+1)) + return hard,good,easy + +@patch +def _passing_early_ivls(self:ReviewSt, ctx): + "Reviewed before due: elapsed days stand in for scheduled, no fuzz" + sched,elap = max(self.ivl,1), self.elapsed + hard = _constrain_passing(ctx, max(elap*ctx.hard_mult, sched*ctx.hard_mult/2), 0, fuzz=False) + good = _constrain_passing(ctx, max(elap*self.ease, sched), 0, fuzz=False) + bonus = ctx.easy_mult - (ctx.easy_mult-1.0)/2 + easy = _constrain_passing(ctx, max(elap*self.ease, sched)*bonus, 0, fuzz=False) + return hard,good,easy + +@patch +def _failing_ivl(self:ReviewSt, ctx): + if ctx.fsrs is not None: return ctx.fsrs[0].ivl # in FSRS, fuzz applies when leaving relearning + lo,hi = _min_max(ctx, ctx.min_lapse_ivl) + return with_fuzz(ctx.fuzz, max(self.ivl,1)*ctx.lapse_mult, lo, hi) + +@patch +def next_answers(self:ReviewSt, ctx): + hard,good,easy = self._passing_ivls(ctx) + lapses = self.lapses+1 + fail = self._failing_ivl(ctx) + days = max(_round(max(fail,0)), 1) + again_review = ReviewSt(days, max(self.ease+EASE_AGAIN, MIN_EASE), lapses, mem=_mem(ctx,1), + leeched=leech_threshold_met(lapses, ctx.leech_threshold)) + if ctx.relearn_steps: + again = RelearnSt(LearnSt(len(ctx.relearn_steps), step_again_delay(ctx.relearn_steps), mem=_mem(ctx,1)), again_review) + elif ctx.fsrs is not None and ctx.allow_short and (ctx.short_steps or not ctx.relearn_steps) and fail < 0.5: + again = RelearnSt(LearnSt(0, int(fail*86400), mem=_mem(ctx,1)), again_review) + else: again = again_review + return Answers(self, again, ReviewSt(hard, max(self.ease+EASE_HARD, MIN_EASE), self.lapses, mem=_mem(ctx,2)), + ReviewSt(good, self.ease, self.lapses, mem=_mem(ctx,3)), + ReviewSt(easy, self.ease+EASE_EASY, self.lapses, mem=_mem(ctx,4))) + +# %% ../nbs/04_scheduler.ipynb #a7371ea7 +def _graduate(ctx, rating): + "Leave the learning steps for review: `grad_good`/`grad_easy` days under SM-2, the model's interval under FSRS" + lo,hi = _min_max(ctx, 1) + if ctx.fsrs is None: + ivl = ctx.grad_easy if rating==4 else ctx.grad_good + return ReviewSt(with_fuzz(ctx.fuzz, max(_round(ivl),1), lo, hi), ctx.init_ease) + st = ctx.fsrs[rating-1] + if rating==4: lo = with_fuzz(ctx.fuzz, ctx.fsrs[2].ivl, lo, hi) + 1 # Easy must clear the fuzzed Good interval + return ReviewSt(with_fuzz(ctx.fuzz, max(_round(st.ivl),1), lo, hi), ctx.init_ease, mem=st.mem) + +def _learn_short(ctx, rating, steps): + "The FSRS short-term state when the model wants this answer back the same day, else None" + if ctx.fsrs is None or not ctx.allow_short or not (ctx.short_steps or not steps): return None + st = ctx.fsrs[rating-1] + return st if st.ivl < 0.5 else None + +@patch +def next_answers(self:LearnSt, ctx): + steps = ctx.steps + def grad_or_short(rating, remaining): + s = _learn_short(ctx, rating, steps) + if s is not None: return LearnSt(remaining, int(s.ivl*86400), mem=s.mem) + return _graduate(ctx, rating) + ad = step_again_delay(steps) + again = LearnSt(len(steps), ad, mem=_mem(ctx,1)) if ad is not None else grad_or_short(1, len(steps)) + hd = step_hard_delay(steps, self.remaining) + hard = LearnSt(self.remaining, hd, mem=_mem(ctx,2)) if hd is not None else grad_or_short(2, self.remaining) + gd = step_good_delay(steps, self.remaining) + good = LearnSt(step_remaining_good(steps, self.remaining), gd, mem=_mem(ctx,3)) if gd is not None else grad_or_short(3, self.remaining) + return Answers(self, again, hard, good, _graduate(ctx, 4)) + +@patch +def next_answers(self:NewSt, ctx): + "A new card answers like a learning card that just failed" + return LearnSt(len(ctx.steps), 0).next_answers(ctx)._replace(current=self) + +def _relearn_pass(rl, ctx, rating): + "A passing FSRS answer in relearning: back to review, or another same-day step if the model wants one" + lo,hi = _min_max(ctx, 1) + st = ctx.fsrs[rating-1] + rev = rl.review._replace(ivl=with_fuzz(ctx.fuzz, max(_round(st.ivl),1), lo, hi), mem=st.mem) + if ctx.allow_short and (ctx.short_steps or not ctx.relearn_steps) and st.ivl < 0.5: + rem = rl.learn.remaining if rating==2 else step_remaining_good(ctx.relearn_steps, rl.learn.remaining) + return RelearnSt(rl.learn._replace(remaining=rem, secs=int(st.ivl*86400), elapsed=0, mem=st.mem), rev) + return rev + +@patch +def next_answers(self:RelearnSt, ctx): + steps,rev,fs = ctx.relearn_steps, self.review, ctx.fsrs + fail = rev._failing_ivl(ctx) + days = max(_round(max(fail,0)), 1) + ad = step_again_delay(steps) + if ad is not None: again = RelearnSt(LearnSt(len(steps), ad, mem=_mem(ctx,1)), rev._replace(ivl=days, elapsed=0, mem=_mem(ctx,1))) + elif fs is not None: + lo,hi = _min_max(ctx, 1) + again_rev = rev._replace(ivl=with_fuzz(ctx.fuzz, max(_round(fail),1), lo, hi), mem=fs[0].mem) + if ctx.allow_short and (ctx.short_steps or not steps) and fail < 0.5: + again = RelearnSt(LearnSt(len(steps), int(fail*86400), mem=fs[0].mem), again_rev) + else: again = again_rev + else: again = rev + hd = step_hard_delay(steps, self.learn.remaining) + if hd is not None: hard = RelearnSt(self.learn._replace(secs=hd, elapsed=0, mem=_mem(ctx,2)), rev._replace(elapsed=0, mem=_mem(ctx,2))) + elif fs is not None: hard = _relearn_pass(self, ctx, 2) + else: hard = rev + gd = step_good_delay(steps, self.learn.remaining) + if gd is not None: + good = RelearnSt(LearnSt(step_remaining_good(steps, self.learn.remaining), gd, mem=_mem(ctx,3)), rev._replace(elapsed=0, mem=_mem(ctx,3))) + elif fs is not None: good = _relearn_pass(self, ctx, 3) + else: good = rev + if fs is not None: + lo,hi = _min_max(ctx, 1) + lo = with_fuzz(ctx.fuzz, fs[2].ivl, lo, hi) + 1 + easy = rev._replace(ivl=with_fuzz(ctx.fuzz, max(_round(fs[3].ivl),1), lo, hi), elapsed=0, mem=fs[3].mem) + else: easy = rev._replace(ivl=rev.ivl+1, elapsed=0) + return Answers(self, again, hard, good, easy) + +# %% ../nbs/04_scheduler.ipynb #01548bec +def _card_data(c): + "The card's `data` column as a dict" + try: return json.loads(c.data) if c.data else {} + except ValueError: return {} + +def card_state(c, cfg, today, now=None): + "The scheduling state of card row `c` under preset `cfg`" + now = ifnone(now, int(time.time())) + left = c.left%1000 + d = _card_data(c) + mem = MemSt(d['s'], d['d']) if 's' in d and 'd' in d else None + if c.type==0: return NewSt(max(c.due,0)) + if c.type==2: + due = min(c.due, today) + return ReviewSt(c.ivl, c.factor/1000, c.lapses, elapsed=max(c.ivl-(due-today), 0), mem=mem) + steps = list(cfg.learn_steps if c.type==1 else cfg.relearn_steps) + last = step_current_delay(steps, left) + if c.queue==1: elapsed = now - (c.due - learn_fuzz(c.id+c.reps-1 if c.reps else None, last)) + elif c.queue==3: elapsed = (today - c.due + max(last//DAY, 1))*DAY + else: elapsed = 0 + learn = LearnSt(left, last, elapsed, mem=mem) + if c.type==1: return learn + return RelearnSt(learn, ReviewSt(c.ivl, c.factor/1000, c.lapses, elapsed=c.ivl, mem=mem)) + +# %% ../nbs/04_scheduler.ipynb #290dbd5f +RLog = namedtuple('RLog', 'id ease ivl lastIvl factor type') + +@patch +def fsrs_on(self:Collection): + "Is FSRS enabled for this collection?" + return bool(self.conf('fsrs', False)) + +def preset_params(cfg): + "The preset's raw FSRS parameters: version 6, else 5, else 4.5, else [] meaning the defaults" + return list(cfg.fsrs_params_6 or cfg.fsrs_params_5 or cfg.fsrs_params_4) + +def allow_short_term(raw): + "May FSRS schedule same-day steps? Requires non-zero short-term params (default params qualify)" + if not raw: return True + return raw[17]>0 and raw[18]>0 if len(raw)>=19 else False + +def ignore_revlogs_before(cfg): + "The preset's ignore-revlogs-before date as epoch ms (0 if unset)" + s = cfg.ignore_revlogs_before_date + return int(datetime.strptime(s, '%Y-%m-%d').replace(tzinfo=timezone.utc).timestamp()*1000) if s else 0 + +@patch +def _deck_dr(self:Collection, did, cfg): + "Effective desired retention: the deck's own override, else the preset's" + k = decks_pb2.Deck.KindContainer() + k.ParseFromString(self.q1('select kind from decks where id=?', did)) + return k.normal.desired_retention if k.normal.HasField('desired_retention') else cfg.desired_retention + +# %% ../nbs/04_scheduler.ipynb #6546e9b9 +def _rl_days(e, next_day_at): return max(next_day_at - e.id//1000, 0)//86400 + +def _fsrs_reviews(entries, next_day_at, ignore_before=0): + "Filter a card's revlog rows for FSRS and compute (rating, delta_t) pairs; None if nothing usable" + first_learn = first_grade = None + for i in reversed(range(len(entries))): + e = entries[i] + if e.type==3 and e.factor==0: continue # cramming + if e.ease>0 and e.id>ignore_before and (e.ivl>=1 or e.ivl<=-86400): first_grade = i + if e.ease>0 and e.type==0: first_learn = i + elif e.type==4 and e.factor==0: # reset + if first_learn is None and first_grade is None: return None + break + elif first_learn is not None: break + complete = first_learn is not None + if complete and entries[first_learn].id < ignore_before and first_learn < len(entries)-1: + complete,first_learn = False,None + start = first_learn if first_learn is not None else first_grade + if start is None: return None + kept = [e for e in entries[start:] if e.ease>0 and not (e.type==3 and e.factor==0)] + if not kept: return None + ds = [0] + [_rl_days(a, next_day_at)-_rl_days(b, next_day_at) for a,b in zip(kept, kept[1:])] + return [(e.ease, dt) for e,dt in zip(kept, ds)], complete, kept + +def _memory_from_revlog(w, entries, next_day_at, historical_retention=0.9, ignore_before=0): + "Replay a card's revlog into a `MemSt`, starting a truncated history from an SM-2 approximation" + out = _fsrs_reviews(entries, next_day_at, ignore_before) + if out is None: return None + revs,complete,kept = out + mem = None + if not complete: + first = kept[0] + ease = (first.factor or 2500)/1000 + mem = memory_state_from_sm2(w, ease, max(first.ivl, 1), historical_retention) + if ease <= 1.1: mem = mem._replace(difficulty=f32((ease-0.1)*9 + 1)) # entry was written by FSRS itself + revs = revs[1:] + for rating,dt in revs: mem = step(w, dt, rating, mem) + return mem + +# %% ../nbs/04_scheduler.ipynb #18d38eb4 +@patch +def _fsrs_ctx(self:Collection, c, cfg, next_day): + "(four FSRS next-states, desired retention, decay) for card `c`, rebuilding memory from the revlog when absent" + raw = preset_params(cfg) + w = fsrs_params(raw) + d = _card_data(c) + mem = MemSt(d['s'], d['d']) if 's' in d and 'd' in d else None + if mem is None and c.type!=0: + rows = [RLog(*r) for r in self.q('select id, ease, ivl, lastIvl, factor, type from revlog where cid=? order by id', c.id)] + mem = _memory_from_revlog(w, rows, next_day, cfg.historical_retention, ignore_revlogs_before(cfg)) + lrt = d.get('lrt') or self.q1('select max(id)/1000 from revlog where cid=? and ease between 1 and 4 and (type!=3 or factor!=0)', c.id) + days = max(next_day-lrt, 0)//86400 if lrt else 0 + dr = self._deck_dr(c.did, cfg) + return next_states(w, mem, dr, days), dr, param_decay(raw) + +# %% ../nbs/04_scheduler.ipynb #fb9dfbdd +@patch +def _card_answers(self:Collection, cid, fuzz, now): + "Load card `cid` fresh and compute its state, the four answer outcomes, and FSRS extras" + c = Card(*self.q('select * from cards where id=?', cid)[0]) + assert not c.odid, "cards in filtered decks are not supported" + now = ifnone(now, int(time.time())) + today,next_day = self.timing() + cfg = self.deck_conf(c.did) + fs = dr = decay = None + if self.fsrs_on(): + fs,dr,decay = self._fsrs_ctx(c, cfg, next_day) + raw = preset_params(cfg) + ctx = mk_ctx(cfg, fuzz=fuzz_factor(c.id, c.reps) if fuzz else None, fsrs=fs, allow_short=allow_short_term(raw), + short_steps=bool(self.conf('fsrsShortTermWithStepsEnabled', False))) + else: ctx = mk_ctx(cfg, fuzz=fuzz_factor(c.id, c.reps) if fuzz else None) + cur = card_state(c, cfg, today, now) + return c, cfg, cur, cur.next_answers(ctx), today, max(next_day-now, 0), now, dr, decay + +@patch +def answer_buttons(self:Collection, card, fuzz=True, now=None): + "For each ease 1-4, `(next_state, delay_secs)` -- what a client shows on its answer buttons" + _,_,_,ans,_,sur,_,_,_ = self._card_answers(card.id if isinstance(card,Card) else card, fuzz, now) + return [(st, ivl_secs(ivl_days(st.ivl_kind(), sur))) for st in ans[1:]] + +# %% ../nbs/04_scheduler.ipynb #a13556f3 +@patch +def _leech_note(self:Collection, nid, now): + tags = self.q1('select tags from notes where id=?', nid).split() + if 'leech' not in tags: + self.con.execute('update notes set tags=?, mod=?, usn=-1 where id=?', (f" {' '.join(tags+['leech'])} ", now, nid)) + +@patch +def _bump_deck_stats(self:Collection, did, today, new_delta, rev_delta, ms_delta): + "Add an answer to the daily counters of deck `did` and its parents, resetting them on a new day" + parts = self.q1('select name from decks where id=?', did).split('\x1f') + names = ['\x1f'.join(parts[:i]) for i in range(1, len(parts)+1)] + for did_,blob in self.q(f"select id, common from decks where name in ({','.join('?'*len(names))})", *names): + c = decks_pb2.Deck.Common() + c.ParseFromString(blob) + if c.last_day_studied != today: + c.new_studied,c.learning_studied,c.review_studied,c.milliseconds_studied = 0,0,0,0 + c.last_day_studied = today + c.new_studied += new_delta + c.review_studied += rev_delta + c.milliseconds_studied += ms_delta + self.con.execute('update decks set common=?, mtime_secs=?, usn=-1 where id=?', (c.SerializeToString(), int(time.time()), did_)) + +_GATHER_ORD = {1:0, 4:0, 3:1, 2:2, 0:3} # queue -> gather order: intraday learn, interday learn, review, new + +@patch +def _bury_siblings(self:Collection, c, cfg, now): + "Bury (queue -2) siblings per the preset, only in queues gathered after the answered card's" + g = _GATHER_ORD.get(c.queue, 99) + qs = [q for q,want in [(0,cfg.bury_new), (2,cfg.bury_reviews and g<=2), (3,cfg.bury_interday_learning and g<=1)] if want] + if qs: self.con.execute(f"update cards set queue=-2, mod=?, usn=-1 where nid=? and id!=? and queue in ({','.join('?'*len(qs))})", + (now, c.nid, c.id, *qs)) + +# %% ../nbs/04_scheduler.ipynb #25b72dfe +def _shifted_d(mem): + "FSRS difficulty normalized to the 0.1-1.1 range revlog factors use, x1000" + return _round(((mem.difficulty-1)/9 + 0.1)*1000) + +@patch +def answer_card(self:Collection, card, ease, taken_ms=0, fuzz=True, now=None): + "Answer `card` (a `Card` or id) with `ease` (1=Again 2=Hard 3=Good 4=Easy), returning the updated `Card`" + cid = card.id if isinstance(card,Card) else card + assert ease in (1,2,3,4), f"ease must be 1-4, got {ease}" + with self._tx(): + c,cfg,cur,ans,today,sur,now,dr,decay = self._card_answers(cid, fuzz, now) + nxt = ans[ease] + taken = min(taken_ms, cfg.cap_answer_time_to_secs*1000) + mem = nxt.learn.mem if isinstance(nxt, RelearnSt) else nxt.mem + d = _card_data(c) + if isinstance(cur, NewSt): d['pos'] = cur.position + d['lrt'] = now + for k in ('s','d','dr'): d.pop(k, None) + if mem is not None: d['s'],d['d'] = round(mem.stability, 4), round(mem.difficulty, 3) + if dr is not None: d['dr'] = round(dr, 2) + if decay is not None: d['decay'] = round(decay, 3) + cols = dict(mod=now, usn=-1, reps=c.reps+1, data=json.dumps(d, separators=(',',':'))) + rl_fact = 0 + if isinstance(nxt, ReviewSt): + cols.update(type=2, queue=2, ivl=nxt.ivl, due=today+nxt.ivl, factor=_round(nxt.ease*1000), lapses=nxt.lapses, left=0) + rl_fact = _shifted_d(mem) if mem is not None else cols['factor'] + else: + learn = nxt if isinstance(nxt, LearnSt) else nxt.learn + if isinstance(nxt, LearnSt): + cols.update(type=1, left=learn.remaining) + rl_fact = _shifted_d(mem) if mem is not None else 0 + else: + cols.update(type=3, left=learn.remaining, ivl=nxt.review.ivl, lapses=nxt.review.lapses, factor=_round(nxt.review.ease*1000)) + rl_fact = _shifted_d(mem) if mem is not None else cols['factor'] + iv = ivl_days(learn.ivl_kind(), sur) + if iv<0: cols.update(queue=1, due=now+learn_fuzz(c.id+c.reps if fuzz else None, -iv)) + else: cols.update(queue=3, due=today+iv) + rev = nxt.review if isinstance(nxt, RelearnSt) else nxt + leeched = isinstance(rev, ReviewSt) and rev.leeched + if leeched and cfg.leech_action==0: cols['queue'] = -1 # LEECH_ACTION_SUSPEND + self.con.execute(f"update cards set {', '.join(f'{k}=?' for k in cols)} where id=?", (*cols.values(), cid)) + self.con.execute('insert into revlog values (?,?,?,?,?,?,?,?,?)', (ts_id(self.con,'revlog'), cid, -1, + ease, ivl_days(nxt.ivl_kind(), sur), ivl_days(cur.ivl_kind(), sur), rl_fact, taken, cur.revlog_kind())) + self._bump_deck_stats(c.did, today, int(c.queue==0), int(c.queue in (2,3)), taken) + if leeched: self._leech_note(c.nid, now) + self._bury_siblings(c, cfg, now) + self._dirty() + return Card(*self.q('select * from cards where id=?', cid)[0]) + +# %% ../nbs/04_scheduler.ipynb #12becb8a +@patch +def unbury_if_day_changed(self:Collection): + "Restore buried cards once the day has rolled over since the last unbury" + today,last = self.today(), self.conf('lastUnburied', 0) + if last < today or today+7 < last: + for cid,typ,due in self.q('select id, type, due from cards where queue in (-2,-3)'): + q = {0:0, 2:2}.get(typ, 1 if due>1_000_000_000 else 3) + self.con.execute('update cards set queue=? where id=?', (q, cid)) + self.con.execute("insert or replace into config values ('lastUnburied',-1,?,?)", + (now_ms()//1000, json.dumps(today).encode())) + +@patch +def _day_limits(self:Collection, did, today): + "(new_left, review_left) for deck `did` today" + cfg = self.deck_conf(did) + cmn = decks_pb2.Deck.Common() + cmn.ParseFromString(self.q1('select common from decks where id=?', did)) + new_done,rev_done = (cmn.new_studied,cmn.review_studied) if cmn.last_day_studied==today else (0,0) + new_left,rev_left = max(cfg.new_per_day-new_done, 0), max(cfg.reviews_per_day-rev_done, 0) + if not self.conf('newCardsIgnoreReviewLimit', False): new_left = min(new_left, rev_left) + return new_left, rev_left + +@patch +def next_card(self:Collection, deck=None, fuzz=True): + "The next card to study in `deck` (default the whole collection, limits from 'Default'), or None" + self.unbury_if_day_changed() + today,now = self.today(), int(time.time()) + new_left,rev_left = self._day_limits(self.deck_id(deck) if deck else 1, today) + cond,ps = self._find_sql(deck) + def pick(extra, *xps): + r = self.q(f'select c.* from cards c join notes n on c.nid=n.id where {cond} and {extra} order by c.due, c.id limit 1', *ps, *xps) + return Card(*r[0]) if r else None + c = pick('c.queue=1 and c.due<=?', now) + if not c and rev_left: c = pick('c.queue=3 and c.due<=?', today) + if not c and rev_left: c = pick('c.queue=2 and c.due<=?', today) + if not c and new_left: c = pick('c.queue=0') + if not c: c = pick('c.queue=1 and c.due<=?', now+self.conf('collapseTime',1200)) + return c diff --git a/nbs/01_collection.ipynb b/nbs/01_collection.ipynb index 15ce3d9..c3685c2 100644 --- a/nbs/01_collection.ipynb +++ b/nbs/01_collection.ipynb @@ -40,6 +40,7 @@ "outputs": [], "source": [ "import hashlib, html, random, time, json\n", + "from datetime import datetime, timezone, timedelta\n", "from contextlib import contextmanager\n", "from fastcore.utils import *\n", "from fastanki.schema import *\n", @@ -138,7 +139,18 @@ "execution_count": null, "id": "78ed1c85", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'R&9`J?%YjF'" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "test_eq(field_csum('hello'), 2868168221)\n", "g = guid64()\n", @@ -296,7 +308,18 @@ "execution_count": null, "id": "cd63e672", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "NT('Basic', flds=['Front', 'Back'], 1 templates)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "col.nt('Basic')" ] @@ -623,7 +646,25 @@ "execution_count": null, "id": "f9e3c9f8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "
\n", + "\n", + "**Front**: hola | **Back**: hello | 🏷 spanish\n", + "\n", + "
" + ], + "text/plain": [ + "Note(1784887842154, Front='hola', Back='hello', tags=['spanish'])" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "n = col.add(Front='hola', Back='hello', deck='Spanish::Vocab', tags='spanish')\n", "n" @@ -771,6 +812,124 @@ "test_eq(col.q1(\"select count(*) from graves where type=2\"), 1)" ] }, + { + "cell_type": "markdown", + "id": "426aac05", + "metadata": {}, + "source": [ + "## The day boundary" + ] + }, + { + "cell_type": "markdown", + "id": "3be47479", + "metadata": {}, + "source": [ + "Anki's scheduler counts days, not timestamps: a review card due on day 507 is due when the collection's day counter reaches 507. The day rolls over at a configurable hour (4am by default), so late-night reviews count as the day before, and the count is computed calendar-wise in the collection's *creation* timezone versus the *current* one, so travelling or DST can't shift historical due dates. This ports `sched_timing_today_v2_new` from Anki's `timing.rs` (with the legacy fallback for collections whose `creationOffset` was never set), and `conf` reads the JSON values in the `config` table where these settings live." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a3ad061", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "@patch\n", + "def conf(self:Collection, key, default=None):\n", + " \"Retrieve `key`'s value from the config table, or `default` if absent\"\n", + " return default if (v := self.q1('select val from config where key=?', key)) is None else json.loads(v)\n", + "\n", + "def sched_timing(\n", + " crt, # Collection creation stamp (epoch secs)\n", + " now, # Current time (epoch secs)\n", + " crt_mins_west=None, # UTC offset at creation (`creationOffset` config; None = legacy collection)\n", + " now_mins_west=0, # Current UTC offset, e.g. from `day_offset`\n", + " rollover=4, # Hour of day when the next day starts\n", + "):\n", + " \"Anki's `(days_elapsed, next_day_at)`: the scheduler day counter and next rollover time\"\n", + " ndt = datetime.fromtimestamp(now, timezone(timedelta(minutes=-now_mins_west)))\n", + " roll = ndt.replace(hour=rollover%24, minute=0, second=0, microsecond=0)\n", + " if crt_mins_west is None:\n", + " crt_roll = datetime.fromtimestamp(crt, ndt.tzinfo).replace(hour=rollover%24, minute=0, second=0, microsecond=0)\n", + " days = (now - int(crt_roll.timestamp()))//86400\n", + " passed = roll < ndt # the legacy path holds the cutoff for the whole rollover second\n", + " else:\n", + " cdt = datetime.fromtimestamp(crt, timezone(timedelta(minutes=-crt_mins_west)))\n", + " passed = roll <= ndt\n", + " days = (ndt.date() - cdt.date()).days - (not passed)\n", + " return max(days,0), int(roll.timestamp()) + 86400*passed\n", + "\n", + "@patch\n", + "def timing(self:Collection):\n", + " \"`(days_elapsed, next_day_at)` for this collection, resolving its config\"\n", + " return sched_timing(self.q1('select crt from col'), int(time.time()), self.conf('creationOffset'),\n", + " day_offset(), min(self.conf('rollover',4),23))\n", + "\n", + "@patch\n", + "def today(self:Collection):\n", + " \"Days since collection creation, Anki's day counter\"\n", + " return self.timing()[0]" + ] + }, + { + "cell_type": "markdown", + "id": "bc62b97c", + "metadata": {}, + "source": [ + "These cases are ported from `timing.rs`'s own tests: a collection created at midnight MDT and reviewed 16 months later from MST (the answer must not shift with the DST change), and the 4am boundary walked second by second:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d1d9b6a", + "metadata": {}, + "outputs": [], + "source": [ + "MDT,MST = 360,420 # minutes west\n", + "def _ts(y,mo,d,h,mins_west): return int(datetime(y,mo,d,h, tzinfo=timezone(timedelta(minutes=-mins_west))).timestamp())\n", + "crt = _ts(2018,8,6,0,MDT)\n", + "test_eq(sched_timing(crt, _ts(2019,12,26,20,MST), MDT, MST)[0], 507)\n", + "test_eq(sched_timing(crt, _ts(2019,12,26,20,MST), MDT, MDT)[0], 507) # DST change must not alter the count\n", + "crt3 = _ts(2018,8,6,3,MDT) # created 3am, so day 1 starts 4am on the 7th\n", + "test_eq(sched_timing(crt3, _ts(2018,8,9,3,MST)+3599, MDT, MST)[0], 2)\n", + "test_eq(sched_timing(crt3, _ts(2018,8,9,4,MST), MDT, MST)[0], 3)\n", + "test_eq(sched_timing(crt, crt-86400, MDT, MDT)[0], 0) # days can't go negative\n", + "d,nda = sched_timing(crt, _ts(2018,8,7,2,MDT), MDT, MDT)\n", + "test_eq((d,nda), (0, _ts(2018,8,7,4,MDT))) # 2am is still day 0; next rollover 4am same morning\n", + "four = _ts(2018,8,9,4,MDT) # exactly at the rollover instant:\n", + "test_eq(sched_timing(crt, four, MDT, MDT)[1], four+86400) # modern timing has rolled over...\n", + "test_eq(sched_timing(crt, four, None, MDT), (3, four)) # ...but the legacy no-creationOffset path has not" + ] + }, + { + "cell_type": "markdown", + "id": "af029b3a", + "metadata": {}, + "source": [ + "And the live check: Anki's own scheduler agrees on today's number and the next cutoff for a collection it created." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07ebe392", + "metadata": {}, + "outputs": [], + "source": [ + "tdt = Path(tempfile.mkdtemp())\n", + "toc = AnkiCollection(str(tdt/'t.anki2'))\n", + "tcol = Collection(tdt/'t.anki2')\n", + "test_eq(tcol.today(), toc.sched.today)\n", + "test_eq(tcol.timing()[1], toc.sched.day_cutoff)\n", + "tcol.close(); toc.close()" + ] + }, { "cell_type": "markdown", "id": "010620c0", @@ -864,14 +1023,17 @@ "outputs": [], "source": [ "class Card:\n", - " def __init__(self, id, nid, did, ord, mod, usn, type, queue, due, ivl): store_attr()\n", + " def __init__(self, id, nid, did, ord, mod, usn, type, queue, due, ivl, factor=0, reps=0, lapses=0, left=0, odue=0, odid=0, flags=0, data=\"\"): store_attr()\n", " def __repr__(self): return f\"Card({self.id}, nid={self.nid}, due={self.due}, ivl={self.ivl}, queue={self.queue})\"\n", - " def _repr_markdown_(self): return f\"Card {self.id} (nid: {self.nid}, due: {self.due}, ivl: {self.ivl}d, queue: {self.queue})\"\n", - "\n", - "@patch\n", - "def today(self:Collection):\n", - " \"Days since collection creation, Anki's day counter\"\n", - " return (int(time.time()) - self.q1('select crt from col'))//86400\n", + " def _repr_markdown_(self):\n", + " if self.queue==0: st = f'new #{self.due}'\n", + " elif self.queue==1:\n", + " secs = max(self.due-int(time.time()), 0)\n", + " st = f\"learning, due in {f'{-(-secs//3600)}h' if secs>=3600 else f'{-(-secs//60)}m'}\" if secs else 'learning, due now'\n", + " elif self.queue==3: st = f'learning, due day {self.due}'\n", + " elif self.queue==2: st = f'review, ivl {self.ivl}d, due day {self.due}'\n", + " else: st = {-1:'suspended'}.get(self.queue, 'buried')\n", + " return f\"Card {self.id} (nid: {self.nid}): {st}\"\n", "\n", "@patch\n", "def find_cards(self:Collection, deck=None, tag=None, added_days=None, is_due=None, where=None, args=(), **fields):\n", @@ -879,8 +1041,10 @@ " nids = None\n", " if fields: nids = {x.id for x in self.find_notes(deck=deck, tag=tag, added_days=added_days, **fields)}\n", " cond,ps = self._find_sql(deck, tag, added_days, where, args)\n", - " if is_due: cond += f' and (c.queue=1 and c.due<=? or c.queue in (2,3) and c.due<=?)'; ps += [int(time.time())+1200, self.today()]\n", - " sql = f'select c.id, c.nid, c.did, c.ord, c.mod, c.usn, c.type, c.queue, c.due, c.ivl from cards c join notes n on c.nid=n.id where {cond} order by c.id'\n", + " if is_due:\n", + " cond += f' and (c.queue=1 and c.due<=? or c.queue in (2,3) and c.due<=?)'\n", + " ps += [int(time.time())+self.conf('collapseTime',1200), self.today()]\n", + " sql = f'select c.* from cards c join notes n on c.nid=n.id where {cond} order by c.id'\n", " return [Card(*r) for r in self.q(sql, *ps) if nids is None or r[1] in nids]\n", "\n", "@patch\n", @@ -892,7 +1056,25 @@ "execution_count": null, "id": "5ccc573c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "
\n", + "\n", + "Card 1784904581422 (nid: 1784904581422): new #1\n", + "\n", + "
" + ], + "text/plain": [ + "Card(1784904581422, nid=1784904581422, due=1, ivl=0, queue=0)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "cards = col.find_cards(deck='Spanish')\n", "test_eq(len(cards), 1)\n", @@ -943,7 +1125,7 @@ " cond,ps = self._find_sql(deck)\n", " sql = ('select sum(c.queue=0), sum(c.queue=1 and c.due<=? or c.queue=3 and c.due<=?), sum(c.queue=2 and c.due<=?) '\n", " f'from cards c join notes n on c.nid=n.id where {cond}')\n", - " r = self.q(sql, int(time.time())+1200, self.today(), self.today(), *ps)[0]\n", + " r = self.q(sql, int(time.time())+self.conf('collapseTime',1200), self.today(), self.today(), *ps)[0]\n", " return tuple(x or 0 for x in r)" ] }, @@ -1018,7 +1200,15 @@ ] } ], - "metadata": {}, + "metadata": { + "solveit": { + "default_code": true, + "mode": "learning", + "use_thinking": true, + "use_tools": true, + "ver": 2 + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/nbs/03_fsrs.ipynb b/nbs/03_fsrs.ipynb new file mode 100644 index 0000000..9d9039a --- /dev/null +++ b/nbs/03_fsrs.ipynb @@ -0,0 +1,374 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "f2b6f3bd", + "metadata": {}, + "outputs": [], + "source": [ + "#| default_exp fsrs" + ] + }, + { + "cell_type": "markdown", + "id": "3ab9abcb", + "metadata": {}, + "source": [ + "# FSRS\n", + "\n", + "> The FSRS-6 memory model, ported from the fsrs-rs crate that Anki embeds" + ] + }, + { + "cell_type": "markdown", + "id": "aba44573", + "metadata": {}, + "source": [ + "FSRS (Free Spaced Repetition Scheduler) models each card as a *memory state*, comprising a stability `S` (the interval, in days, at which recall probability falls to 90%) and a difficulty `D` in 1-10. A power-law forgetting curve — with a per-user shape parameter, `w20` — gives the probability of recall after any elapsed time. The next interval is wherever that curve crosses the user's desired retention, and each review updates `(S, D)` by the formulas below.\n", + "\n", + "The heavy machinery in FSRS (the optimizer that fits the 21 parameters to a user's history) lives in the clients and reaches us as numbers in the deck preset, so fastanki doesn't need it.\n", + "\n", + "This module is a direct port of the scheduling half of [fsrs-rs](https://github.com/open-spaced-repetition/fsrs-rs) (the crate compiled into Anki itself). The `scheduler` module wires it to the state machine. Collections in the wild carry parameters from three FSRS generations — 17 (FSRS-4.5, the `fsrsWeights` sync field), 19 (FSRS-5) or 21 (FSRS-6) numbers — and `fsrs_params` upgrades and range-clips them exactly as `FSRS::new` does, so a user who last optimized on an older Anki schedules identically here.\n", + "\n", + "Note, the rust crate computes in 32-bit floats, Python in 64-bit. We squeeze each formula's *output* through float32 (`f32`), which keeps parity for everything that lands in the collection; Anki's own test suite acknowledges float variance across platforms and rounds before comparing, and the oracle tests in the scheduler notebook do the same." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae880466", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "import math\n", + "from struct import pack, unpack\n", + "from collections import namedtuple\n", + "from fastcore.utils import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40a562dc", + "metadata": {}, + "outputs": [], + "source": [ + "from fastcore.test import *" + ] + }, + { + "cell_type": "markdown", + "id": "9c410ead", + "metadata": {}, + "source": [ + "## Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2404bf38", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "S_MIN,S_MAX,D_MIN,D_MAX = 0.001,36500.0,1.0,10.0\n", + "FSRS5_DECAY,FSRS6_DECAY = 0.5,0.1542\n", + "\n", + "DEFAULT_PARAMS = [0.212,1.2931,2.3065,8.2956,6.4133,0.8334,3.0194,0.001,1.8722,0.1666,0.796,\n", + " 1.4835,0.0614,0.2629,1.6483,0.6014,1.8729,0.5425,0.0912,0.0658,FSRS6_DECAY]\n", + "\n", + "_CLAMPS = [(S_MIN,100.)]*4 + [(D_MIN,D_MAX),(0.001,4.0),(0.001,4.0),(0.001,0.75),(0.,4.5),(0.,0.8),(0.001,3.5),\n", + " (0.001,5.0),(0.001,0.25),(0.001,0.9),(0.,4.0),(0.,1.0),(1.0,6.0),(0.,2.0),(0.,2.0),(0.,0.8),(0.1,0.8)]\n", + "\n", + "def f32(x): return unpack('f', pack('f', x))[0]\n", + "def clamp(x, lo, hi): return min(max(x,lo),hi)\n", + "\n", + "def fsrs_params(w):\n", + " \"Upgrade a 0/17/19/21-length parameter list to FSRS-6's 21 numbers and clip to legal ranges, like `FSRS::new`\"\n", + " w = [float(x) for x in w] or list(DEFAULT_PARAMS)\n", + " if len(w)==17:\n", + " w[4],w[5],w[6] = w[4]+2*w[5], math.log(w[5]*3+1)/3, w[6]+0.5\n", + " w += [0.,0.,0.,FSRS5_DECAY]\n", + " elif len(w)==19: w += [0.,FSRS5_DECAY]\n", + " assert len(w)==21, f\"invalid FSRS parameter count: {len(w)}\"\n", + " return [f32(clamp(x,*b)) for x,b in zip(w,_CLAMPS)]\n", + "\n", + "def param_decay(w):\n", + " \"The forgetting-curve decay for a *raw* (pre-upgrade) parameter list, Anki's `get_decay_from_params`\"\n", + " return FSRS6_DECAY if not len(w) else (FSRS5_DECAY if len(w)<21 else w[20])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d51e9a0", + "metadata": {}, + "outputs": [], + "source": [ + "test_eq(len(fsrs_params([])), 21)\n", + "test_eq(fsrs_params([]), [f32(x) for x in DEFAULT_PARAMS])\n", + "up = fsrs_params([0.4,0.6,2.4,5.8,4.93,0.94,0.86,0.01,1.49,0.14,0.94,2.18,0.05,0.34,1.26,0.29,2.61])\n", + "test_eq(len(up), 21)\n", + "test_close(up[4], 4.93+2*0.94, eps=1e-4) # FSRS-4.5's difficulty params shift on upgrade\n", + "test_close(up[20], FSRS5_DECAY, eps=1e-6) # ...and keep the old fixed decay\n", + "test_eq(fsrs_params([0.1]*19)[18:], [f32(0.1), 0.0, FSRS5_DECAY]) # 19-param sets gain w19=0 and the fixed FSRS-5 decay\n", + "test_eq(param_decay([]), FSRS6_DECAY)\n", + "test_eq(param_decay([0.1]*17), FSRS5_DECAY)\n", + "test_eq(param_decay(DEFAULT_PARAMS), FSRS6_DECAY)" + ] + }, + { + "cell_type": "markdown", + "id": "7aa95e1e", + "metadata": {}, + "source": [ + "## The forgetting curve" + ] + }, + { + "cell_type": "markdown", + "id": "d95b2e29", + "metadata": {}, + "source": [ + "By construction, retrievability is exactly 0.9 when the elapsed time equals the stability, whatever the decay — and the next interval inverts the curve, so at desired retention 0.9, the interval *is* the stability:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cdaadaa6", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "def forgetting_curve(w, t, s):\n", + " \"Probability of recall `t` days after a review that left stability `s`\"\n", + " decay = -w[20]\n", + " factor = math.exp(math.log(0.9)/decay) - 1\n", + " return f32((t/s*factor + 1)**decay)\n", + "\n", + "def next_interval(w, s, dr):\n", + " \"The (fractional) days until retrievability falls to desired retention `dr`, at stability `s`\"\n", + " decay = -w[20]\n", + " factor = math.exp(math.log(0.9)/decay) - 1\n", + " return f32(s/factor*(dr**(1/decay) - 1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74db9c11", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.742717444896698, 0.972769558429718)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "w = fsrs_params([])\n", + "test_close(forgetting_curve(w, 5, 5), 0.9, eps=1e-6)\n", + "test_close(next_interval(w, 5, 0.9), 5, eps=1e-5)\n", + "assert next_interval(w, 5, 0.95) < 5 < next_interval(w, 5, 0.8) # higher retention -> shorter intervals\n", + "forgetting_curve(w, 30, 5), forgetting_curve(w, 1, 5)" + ] + }, + { + "cell_type": "markdown", + "id": "30cb3aaa", + "metadata": {}, + "source": [ + "## Memory state updates" + ] + }, + { + "cell_type": "markdown", + "id": "0f50427e", + "metadata": {}, + "source": [ + "`step` is one review: a rating (1=Again, ..., 4=Easy) after `delta_t` days. The first rating of a new card reads its initial state straight from the first four parameters (stability) and `w4`/`w5` (difficulty). After that:\n", + "- difficulty moves by the rating with linear damping and reversion toward the Easy-init mean\n", + "- a pass multiplies stability by a factor that grows with ease of recall (and shrinks under `w15` for Hard, grows under `w16` for Easy)\n", + "- a lapse rebuilds stability from scratch, capped so it can't exceed the pre-lapse value\n", + "- and a *same-day* review (`delta_t` 0) uses the separate short-term formula in `w17`-`w19`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ced935c2", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "class MemSt(namedtuple('MemSt', 'stability difficulty')):\n", + " \"An FSRS memory state: `stability` in days, `difficulty` in 1-10\"\n", + "\n", + "def _init_d(w, r): return w[4] - math.exp(w[5]*(r-1)) + 1\n", + "\n", + "def _next_d(w, d, r):\n", + " nd = d + (10-d)*(-w[6]*(r-3))/9\n", + " return clamp(w[7]*(_init_d(w,4)-nd)+nd, D_MIN, D_MAX)\n", + "\n", + "def _s_success(w, s, d, r, rating):\n", + " hp = w[15] if rating==2 else 1.0\n", + " eb = w[16] if rating==4 else 1.0\n", + " return s*(math.exp(w[8])*(11-d)*s**-w[9]*(math.exp((1-r)*w[10])-1)*hp*eb + 1)\n", + "\n", + "def _s_fail(w, s, d, r):\n", + " ns = w[11]*d**-w[12]*((s+1)**w[13]-1)*math.exp((1-r)*w[14])\n", + " return min(ns, s/math.exp(w[17]*w[18]))\n", + "\n", + "def _s_short_term(w, s, rating):\n", + " \"Same-day stability. Floors the multiplier at 1 for Good/Easy only\"\n", + " sinc = math.exp(w[17]*(rating-3+w[18]))*s**-w[19]\n", + " return s*(max(sinc, 1.0) if rating>=3 else sinc)\n", + "\n", + "def step(w, delta_t, rating, mem):\n", + " \"Memory state after rating a card `delta_t` days since its last review (`mem` None: first rating of a new card)\"\n", + " if mem is None:\n", + " r = clamp(rating, 1, 4)\n", + " return MemSt(f32(clamp(w[r-1], S_MIN, S_MAX)), f32(clamp(_init_d(w,r), D_MIN, D_MAX)))\n", + " s,d = clamp(mem.stability, S_MIN, S_MAX), clamp(mem.difficulty, D_MIN, D_MAX)\n", + " r = forgetting_curve(w, delta_t, s)\n", + " if delta_t==0: ns = _s_short_term(w, s, rating)\n", + " elif rating==1: ns = _s_fail(w, s, d, r)\n", + " else: ns = _s_success(w, s, d, r, rating)\n", + " return MemSt(f32(clamp(ns, S_MIN, S_MAX)), f32(_next_d(w, d, rating)))\n", + "\n", + "ItemSt = namedtuple('ItemSt', 'mem ivl')\n", + "\n", + "def next_states(w, mem, dr, days_elapsed):\n", + " \"For each rating 1-4 (index `[ease-1]`), the next `MemSt` and its desired-retention interval\"\n", + " sts = [step(w, float(days_elapsed), rating, mem) for rating in (1,2,3,4)]\n", + " return [ItemSt(m, next_interval(w, m.stability, dr)) for m in sts]" + ] + }, + { + "cell_type": "markdown", + "id": "288bf38c", + "metadata": {}, + "source": [ + "The check below is fsrs-rs's own doc-test: a new card at desired retention 0.9 lands on these exact states for each button." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "262fcefe", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ItemSt(mem=MemSt(stability=2.30649995803833, difficulty=2.1181039810180664), ivl=2.30649995803833)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ns = next_states(w, None, 0.9, 0)\n", + "for got,(ws,wd) in zip(ns, [(0.212,6.4133),(1.2931,5.1121707),(2.3065,2.118104),(8.2956,1.0)]):\n", + " test_close((got.mem.stability, got.mem.difficulty, got.ivl), (ws, wd, ws), eps=1e-4)\n", + "ns[2]" + ] + }, + { + "cell_type": "markdown", + "id": "fc77506e", + "metadata": {}, + "source": [ + "## Approximating memory state from SM-2" + ] + }, + { + "cell_type": "markdown", + "id": "cd81350f", + "metadata": {}, + "source": [ + "A card that has scheduling state but no usable review history (imported with truncated revlogs, say) gets a memory state approximated from its SM-2 ease and interval, assuming the user's historical retention. The scheduler uses this as the starting state when replaying an incomplete revlog." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2188716f", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "def memory_state_from_sm2(w, ease_factor, interval, sm2_retention=0.9):\n", + " \"A `MemSt` inferred from SM-2 `ease_factor` (eg 2.5) and `interval` days\"\n", + " decay = -w[20]\n", + " factor = 0.9**(1/decay) - 1\n", + " s = max(interval, S_MIN)*factor/(sm2_retention**(1/decay) - 1)\n", + " d = 11 - (ease_factor-1)/(math.exp(w[8])*s**-w[9]*(math.exp((1-sm2_retention)*w[10]) - 1))\n", + " return MemSt(f32(s), f32(clamp(d, D_MIN, D_MAX)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ada57d1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "MemSt(stability=10.0, difficulty=6.914055347442627)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "m = memory_state_from_sm2(w, 2.5, 10)\n", + "test_close(m.stability, 10, eps=1e-4) # at the assumed retention, stability ~= the SM-2 interval\n", + "assert 1 <= m.difficulty <= 10\n", + "m2 = memory_state_from_sm2(w, 1.3, 10)\n", + "assert m2.difficulty > m.difficulty # low ease reads as high difficulty\n", + "m" + ] + } + ], + "metadata": { + "solveit": { + "default_code": true, + "mode": "learning", + "use_thinking": true, + "use_tools": true, + "ver": 2 + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/04_scheduler.ipynb b/nbs/04_scheduler.ipynb new file mode 100644 index 0000000..9bfce2f --- /dev/null +++ b/nbs/04_scheduler.ipynb @@ -0,0 +1,1532 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "522c2526", + "metadata": {}, + "outputs": [], + "source": [ + "#| default_exp scheduler" + ] + }, + { + "cell_type": "markdown", + "id": "10ab5be8", + "metadata": {}, + "source": [ + "# Reviewing\n", + "\n", + "> The v3 scheduler: card states, answer buttons, and the revlog, compatible with every Anki client" + ] + }, + { + "cell_type": "markdown", + "id": "49676761", + "metadata": {}, + "source": [ + "This module is a port of the scheduling half of Anki's Rust `scheduler` module:\n", + "- the state machine that moves cards between new, learning, review and relearning\n", + "- the SM-2 interval arithmetic\n", + "- the bookkeeping an answer leaves behind (the card row, a `revlog` entry, deck daily counters, leech handling)\n", + "\n", + "The review log is the durable source of truth in Anki's design: any client can rebuild scheduling state from it, so the revlog rows we write are what make a fastanki review indistinguishable from one done on e.g. the desktop or mobile app.\n", + "\n", + "Like the rest of fastanki, every claim here is checked against Anki itself: the `anki` package answers the same cards under the same config, and we compare its stored scheduling states, card rows and revlog entries with ours. Setting `ANKI_TEST_MODE` before the oracle loads disables its interval fuzz, making those comparisons exact." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c575cdbb", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "import json, time, random\n", + "from datetime import datetime, timezone\n", + "from collections import namedtuple\n", + "from fastcore.utils import *\n", + "from fastanki.schema import *\n", + "from fastanki.collection import *\n", + "from fastanki._proto import deck_config_pb2, decks_pb2\n", + "from fastanki.fsrs import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0fbc584d", + "metadata": {}, + "outputs": [], + "source": [ + "import os, tempfile, shutil\n", + "from fastcore.test import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "def320cc", + "metadata": {}, + "outputs": [], + "source": [ + "os.environ['ANKI_TEST_MODE'] = '1' # must precede the oracle's first scheduling call: disables its interval fuzz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb70f9df", + "metadata": {}, + "outputs": [], + "source": [ + "from anki.collection import Collection as AnkiCollection\n", + "from anki.scheduler.v3 import CardAnswer" + ] + }, + { + "cell_type": "markdown", + "id": "b52b8e84", + "metadata": {}, + "source": [ + "## Deck presets" + ] + }, + { + "cell_type": "markdown", + "id": "822d2d2c", + "metadata": {}, + "source": [ + "Scheduling is configured per deck preset: learning steps, ease multipliers, daily limits, leech handling. The settings live in the `deck_config` table as a protobuf blob whose field names already say what they mean, so rather than wrap it, we hand back the parsed `DeckConfig.Config` message directly. A deck names its preset inside its `kind` blob." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "682bf3b4", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "@patch\n", + "def deck_conf(self:Collection, did):\n", + " \"The parsed `DeckConfig.Config` preset governing deck `did`\"\n", + " k = decks_pb2.Deck.KindContainer()\n", + " k.ParseFromString(self.q1('select kind from decks where id=?', did))\n", + " blob = self.q1('select config from deck_config where id=?', k.normal.config_id or 1) or default_deck_config()\n", + " c = deck_config_pb2.DeckConfig.Config()\n", + " c.ParseFromString(blob)\n", + " return c" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71e6f02a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1, 2.5, 1.2000000476837158)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "td = Path(tempfile.mkdtemp())\n", + "col = Collection.open(td/'collection.anki2')\n", + "cfg = col.deck_conf(1)\n", + "test_eq(list(cfg.learn_steps), [1.0,10.0])\n", + "test_eq((cfg.leech_threshold, cfg.maximum_review_interval), (8, 36500))\n", + "cfg.graduating_interval_good, cfg.initial_ease, cfg.hard_multiplier" + ] + }, + { + "cell_type": "markdown", + "id": "81a0516b", + "metadata": {}, + "source": [ + "## Card states" + ] + }, + { + "cell_type": "markdown", + "id": "1b614f56", + "metadata": {}, + "source": [ + "A card is always in one of four states, and every answer maps a state to a new state.\n", + "- `NewSt` waits in the new queue at a `position`\n", + "- `LearnSt` is inside the (re)learning steps with `remaining` steps left and `secs` until the next showing\n", + "- `ReviewSt` has graduated and comes back every `ivl` days, its `ease` deciding how fast that grows\n", + "- `RelearnSt` is a failed review: a `LearnSt` to work through plus the `ReviewSt` to return to.\n", + "\n", + "They are named tuples, so states compare by value in tests.\n", + "\n", + "Intervals travel in the revlog's own convention: a non-negative number is days, a negative number is `-seconds`. `ivl_days` is Anki's `maybe_as_days`: a seconds interval that crosses the next day boundary becomes a day count, which is how a `1d` learning step lands in the day-learn queue rather than 24 hours of wall clock." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4fdb2d86", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "class NewSt(namedtuple('NewSt', 'position')):\n", + " \"Not yet studied; `position` orders the new queue\"\n", + " def ivl_kind(self): return 0\n", + " def revlog_kind(self): return 0\n", + "\n", + "class LearnSt(namedtuple('LearnSt', 'remaining secs elapsed mem', defaults=(0,None))):\n", + " \"In the learning steps: `remaining` steps left, `secs` until the next showing\"\n", + " def ivl_kind(self): return -self.secs\n", + " def revlog_kind(self): return 0\n", + "\n", + "class ReviewSt(namedtuple('ReviewSt', 'ivl ease lapses elapsed leeched mem', defaults=(0,0,False,None))):\n", + " \"Graduated: due again in `ivl` days, growing by `ease`\"\n", + " def ivl_kind(self): return self.ivl\n", + " def revlog_kind(self): return 3 if self.elapsed=secs_to_rollover: return (-iv-secs_to_rollover)//86400 + 1\n", + " return iv\n", + "\n", + "def ivl_secs(iv):\n", + " \"An interval in plain seconds\"\n", + " return -iv if iv<0 else iv*86400" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "120c397c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'g'" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_eq(ivl_days(-600, 3600), -600) # 10min tonight stays seconds\n", + "test_eq(ivl_days(-86400, 3600), 1) # a 1d step becomes day-learn: due tomorrow\n", + "test_eq(ivl_days(-90000, 3600), 2) # ...or the day after, if little of today remains\n", + "test_eq(ReviewSt(10, 2.5).revlog_kind(), 3) # reviewed early: logged as kind 3 like Anki\n", + "test_eq(ReviewSt(10, 2.5, elapsed=10).revlog_kind(), 1)\n", + "Answers(*'cahge')[3]" + ] + }, + { + "cell_type": "markdown", + "id": "3b16cc33", + "metadata": {}, + "source": [ + "## Learning steps" + ] + }, + { + "cell_type": "markdown", + "id": "4a453ff1", + "metadata": {}, + "source": [ + "The learning-steps arithmetic ports `steps.rs`. Steps are minutes; a card's `remaining` count says how many are left (the stored `left` column may carry a legacy thousands part, so it's taken mod 1000). Two quirks are Anki's own: on the *first* step, Hard averages the first two steps (or takes 1.5x the only step, capped a day above it) so it lands between Again and Good; and any delay past a day is rounded to whole days so morning and evening study give the same answer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7bf1d31", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "DAY = 86400\n", + "\n", + "def _round(x):\n", + " \"Round half away from zero, like Rust's `round` (Python's `round` is banker's)\"\n", + " return int(x+0.5) if x>=0 else -int(-x+0.5)\n", + "\n", + "def _step_secs(steps, i): return int(steps[i]*60) if 0<=iDAY else secs\n", + "\n", + "def step_again_delay(steps): return _step_secs(steps, 0)\n", + "\n", + "def step_hard_delay(steps, remaining):\n", + " idx = _step_idx(steps, remaining)\n", + " cur = _step_secs(steps, idx) or _step_secs(steps, 0)\n", + " if cur is None: return None\n", + " if idx>0: return cur\n", + " nxt = _step_secs(steps, 1)\n", + " if nxt is not None: return _round_days((cur+nxt)//2)\n", + " return _round_days(min(cur*3//2, cur+DAY))\n", + "\n", + "def step_good_delay(steps, remaining): return _step_secs(steps, _step_idx(steps, remaining)+1)\n", + "def step_current_delay(steps, remaining): return _step_secs(steps, _step_idx(steps, remaining)) or 0\n", + "def step_remaining_good(steps, remaining): return len(steps)-_step_idx(steps, remaining)-1" + ] + }, + { + "cell_type": "markdown", + "id": "cce7c889", + "metadata": {}, + "source": [ + "These vectors are `steps.rs`'s own test suite: a lone 10-minute step, a 3-day step (Hard capped at 4 days), and two- and three-step ladders at each position." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "896f55b0", + "metadata": {}, + "outputs": [], + "source": [ + "def _delays(steps, remaining): return (step_again_delay(steps), step_hard_delay(steps, remaining), step_good_delay(steps, remaining))\n", + "test_eq(_delays([10.0], 1), (600, 900, None))\n", + "test_eq(_delays([3*DAY/60], 1), (3*DAY, 4*DAY, None))\n", + "test_eq(_delays([1.0,10.0], 2), (60, 330, 600))\n", + "test_eq(_delays([1.0,10.0], 1), (60, 600, None))\n", + "test_eq(_delays([1.0,10.0,100.0], 3), (60, 330, 600))\n", + "test_eq(_delays([1.0,10.0,100.0], 2), (60, 600, 6000))\n", + "test_eq(_delays([1.0,10.0,100.0], 1), (60, 6000, None))\n", + "test_eq(step_remaining_good([1.0,10.0], 2), 1)\n", + "test_eq(step_current_delay([1.0,10.0], 1), 600)" + ] + }, + { + "cell_type": "markdown", + "id": "5dd5f8a5", + "metadata": {}, + "source": [ + "## Interval fuzz" + ] + }, + { + "cell_type": "markdown", + "id": "8cf77bee", + "metadata": {}, + "source": [ + "Anki nudges every review interval by a small random amount so cards added together don't stay clumped forever. The fuzz *bounds* port `fuzz.rs` exactly: nothing below 2.5 days, then ±1 day plus a sliding percentage of the days in each range. The *pick* within those bounds comes from a per-`(card, reps)` seeded generator, so re-computing a card's schedule is deterministic.\n", + "\n", + "Note: Rust and Python use different RNGs, so with the same seed the two implementations pick different (equally valid) points inside identical bounds; with fuzz disabled (`ANKI_TEST_MODE`), the two agree exactly. `learn_fuzz` is the separate, smaller fuzz applied to intraday learning delays: up to 25% extra, capped at 5 minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "701a80dc", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "_FUZZ_RANGES = [(2.5,7.0,0.15), (7.0,20.0,0.1), (20.0,None,0.05)]\n", + "\n", + "def fuzz_delta(ivl):\n", + " \"Days of fuzz applied either side of `ivl`\"\n", + " if ivl<2.5: return 0.0\n", + " d = 1.0\n", + " for start,end,f in _FUZZ_RANGES: d += f*max(min(ivl, end or ivl)-start, 0)\n", + " return d\n", + "\n", + "def fuzz_bounds(\n", + " ivl, # Undisturbed interval in days (may be fractional)\n", + " lo=1, # Minimum permitted result\n", + " hi=36500, # Maximum permitted result\n", + "):\n", + " \"Inclusive `(lower, upper)` day bounds for a fuzzed interval, respecting `lo`/`hi`\"\n", + " lo = min(lo, hi)\n", + " ivl = clamp(ivl, lo, hi)\n", + " d = fuzz_delta(ivl)\n", + " l,u = _round(ivl-d), _round(ivl+d)\n", + " l,u = clamp(l, lo, hi), clamp(u, lo, hi)\n", + " if u==l and u>2 and u prev_ivl: return prev_ivl+1\n", + " return prev_ivl if prev_ivl <= upper else 0\n", + "\n", + "def learn_fuzz(fz_seed, secs):\n", + " \"Intraday learning delay with Anki's up-to-25% (max 5min) extension; `fz_seed` None leaves it alone\"\n", + " if fz_seed is None: return secs\n", + " upper = secs + int(min(secs*0.25, 300.0))\n", + " if secs >= upper: return secs\n", + " return random.Random(fz_seed).randrange(secs, upper)" + ] + }, + { + "cell_type": "markdown", + "id": "e5eacf2c", + "metadata": {}, + "source": [ + "`fuzz.rs`'s vectors, driven at fuzz factors 0, 0.5 and 0.99 to hit each bound and the middle:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25f09b4e", + "metadata": {}, + "outputs": [], + "source": [ + "def _lmu(ivl, lo, hi): return tuple(with_fuzz(f, ivl, lo, hi) for f in (0.0, 0.5, 0.99))\n", + "test_eq(with_fuzz(None, 1.5, 1, 100), 2)\n", + "test_eq(with_fuzz(None, 101.0, 1, 100), 100)\n", + "test_eq(_lmu(1.0, 1, 1000), (1,1,1)) # no fuzz below 2.5 days\n", + "test_eq(_lmu(2.5, 1, 1000), (2,3,4)) # then 1 day either side\n", + "test_eq(_lmu(7.0, 1, 1000), (5,7,9)) # plus 0.15/day in 2.5-7\n", + "test_eq(_lmu(17.0, 1, 1000), (14,17,20)) # plus 0.1/day in 7-20\n", + "test_eq(_lmu(37.0, 1, 1000), (33,37,41)) # plus 0.05/day above 20\n", + "test_eq(_lmu(2.0, 2, 1000), (2,2,2))\n", + "test_eq(_lmu(2.0, 3, 1000), (3,4,4)) # widened to a 2-day range when bounds allow\n", + "test_eq(_lmu(2.0, 3, 3), (3,3,3))\n", + "test_eq(_lmu(19.9, 3, 1000), (17,20,23))\n", + "test_eq(learn_fuzz(None, 600), 600)\n", + "assert 600 <= learn_fuzz(42, 600) < 750\n", + "test_eq(learn_fuzz(42, 600), learn_fuzz(42, 600)) # seeded: recomputable" + ] + }, + { + "cell_type": "markdown", + "id": "7e5c5bf3", + "metadata": {}, + "source": [ + "## The state machine" + ] + }, + { + "cell_type": "markdown", + "id": "d9ef9ae3", + "metadata": {}, + "source": [ + "`Ctx` carries what a transition needs from the deck preset, plus the per-answer fuzz factor (`fsrs` stays None for SM-2; a later section fills it). Each state's `next_answers` returns an `Answers` of the four button outcomes, a direct port of `review.rs`, `learning.rs` and `relearning.rs`:\n", + "\n", + "- A failing review multiplies its interval by `lapse_mult` (0 by default: start over), drops ease by 0.2, and enters relearning if there are relearn steps. Lapses at the leech threshold — and every half-threshold after — mark the card `leeched`.\n", + "- A passing review scales by `hard_mult`, `ease`, or `ease * easy_mult`, with overdue days credited at half weight for Good and full weight for Easy, each button forced at least a day past the previous one. Reviewed *early* (`elapsed < ivl`, only reachable through filtered decks — for normal decks `card_state` clamps due to today — but ported for completeness), elapsed days take the place of scheduled days and no fuzz applies.\n", + "- Learning cards walk the steps: Again restarts them, Hard repeats (with the first-step average quirk), Good advances, Easy graduates straight to `grad_easy` days. Past the last step, Good graduates to `grad_good`.\n", + "- New cards answer exactly like a learning card that just failed: full steps remaining." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a57d10b3", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "_CTX = 'fuzz steps relearn_steps grad_good grad_easy init_ease hard_mult easy_mult ivl_mult lapse_mult max_ivl min_lapse_ivl leech_threshold fsrs allow_short short_steps'\n", + "class Ctx(namedtuple('Ctx', _CTX, defaults=(None,False,False))):\n", + " \"Everything a state transition needs: the deck preset's numbers plus this answer's fuzz factor\"\n", + "\n", + "def mk_ctx(cfg, fuzz=None, fsrs=None, **over):\n", + " \"A `Ctx` from deck preset `cfg`, overridable by keyword\"\n", + " d = dict(fuzz=fuzz, steps=list(cfg.learn_steps), relearn_steps=list(cfg.relearn_steps),\n", + " grad_good=cfg.graduating_interval_good, grad_easy=cfg.graduating_interval_easy, init_ease=cfg.initial_ease,\n", + " hard_mult=cfg.hard_multiplier, easy_mult=cfg.easy_multiplier, ivl_mult=cfg.interval_multiplier,\n", + " lapse_mult=cfg.lapse_multiplier, max_ivl=cfg.maximum_review_interval,\n", + " min_lapse_ivl=cfg.minimum_lapse_interval, leech_threshold=cfg.leech_threshold, fsrs=fsrs)\n", + " d.update(over)\n", + " return Ctx(**d)\n", + "\n", + "def _min_max(ctx, minimum):\n", + " hi = max(ctx.max_ivl, 1)\n", + " return clamp(minimum, 1, hi), hi\n", + "\n", + "def leech_threshold_met(lapses, threshold):\n", + " \"True at `threshold` lapses, and every half-threshold (rounded up) after\"\n", + " if not threshold: return False\n", + " half = max(-(-threshold//2), 1)\n", + " return lapses>=threshold and (lapses-threshold)%half==0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "beff5ce2", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "EASE_AGAIN,EASE_HARD,EASE_EASY,MIN_EASE = -0.2,-0.15,0.15,1.3\n", + "\n", + "def _constrain_passing(ctx, ivl, minimum, fuzz=True):\n", + " if ctx.fsrs is None: ivl *= ctx.ivl_mult\n", + " lo,hi = _min_max(ctx, minimum)\n", + " return with_fuzz(ctx.fuzz, ivl, lo, hi) if fuzz else clamp(_round(ivl), lo, hi)\n", + "\n", + "def _mem(ctx, rating):\n", + " \"The FSRS memory state this rating leads to (None under SM-2)\"\n", + " return ctx.fsrs[rating-1].mem if ctx.fsrs is not None else None\n", + "\n", + "@patch\n", + "def _passing_ivls(self:ReviewSt, ctx):\n", + " \"Hard/good/easy intervals, each at least a day past the one before\"\n", + " if ctx.fsrs is not None: return self._passing_fsrs_ivls(ctx)\n", + " if self.elapsed < self.ivl: return self._passing_early_ivls(ctx)\n", + " cur,late = max(self.ivl,1), max(self.elapsed-self.ivl, 0)\n", + " hard_min = 0 if ctx.hard_mult<=1.0 else self.ivl+1\n", + " hard = _constrain_passing(ctx, cur*ctx.hard_mult, hard_min)\n", + " good_min = self.ivl+1 if ctx.hard_mult<=1.0 else hard+1\n", + " good = _constrain_passing(ctx, (cur+late/2)*self.ease, good_min)\n", + " easy = _constrain_passing(ctx, (cur+late)*self.ease*ctx.easy_mult, good+1)\n", + " return hard,good,easy\n", + "\n", + "@patch\n", + "def _passing_fsrs_ivls(self:ReviewSt, ctx):\n", + " \"FSRS intervals come straight from the memory model; fuzz may not shrink an interval that grew\"\n", + " ivls = [s.ivl for s in ctx.fsrs]\n", + " hard = _constrain_passing(ctx, ivls[1], max(min_fuzz_ivl(ivls[1], self.ivl, ctx.max_ivl), 1))\n", + " good = _constrain_passing(ctx, ivls[2], max(min_fuzz_ivl(ivls[2], self.ivl, ctx.max_ivl), hard+1))\n", + " easy = _constrain_passing(ctx, ivls[3], max(min_fuzz_ivl(ivls[3], self.ivl, ctx.max_ivl), good+1))\n", + " return hard,good,easy\n", + "\n", + "@patch\n", + "def _passing_early_ivls(self:ReviewSt, ctx):\n", + " \"Reviewed before due: elapsed days stand in for scheduled, no fuzz\"\n", + " sched,elap = max(self.ivl,1), self.elapsed\n", + " hard = _constrain_passing(ctx, max(elap*ctx.hard_mult, sched*ctx.hard_mult/2), 0, fuzz=False)\n", + " good = _constrain_passing(ctx, max(elap*self.ease, sched), 0, fuzz=False)\n", + " bonus = ctx.easy_mult - (ctx.easy_mult-1.0)/2\n", + " easy = _constrain_passing(ctx, max(elap*self.ease, sched)*bonus, 0, fuzz=False)\n", + " return hard,good,easy\n", + "\n", + "@patch\n", + "def _failing_ivl(self:ReviewSt, ctx):\n", + " if ctx.fsrs is not None: return ctx.fsrs[0].ivl # in FSRS, fuzz applies when leaving relearning\n", + " lo,hi = _min_max(ctx, ctx.min_lapse_ivl)\n", + " return with_fuzz(ctx.fuzz, max(self.ivl,1)*ctx.lapse_mult, lo, hi)\n", + "\n", + "@patch\n", + "def next_answers(self:ReviewSt, ctx):\n", + " hard,good,easy = self._passing_ivls(ctx)\n", + " lapses = self.lapses+1\n", + " fail = self._failing_ivl(ctx)\n", + " days = max(_round(max(fail,0)), 1)\n", + " again_review = ReviewSt(days, max(self.ease+EASE_AGAIN, MIN_EASE), lapses, mem=_mem(ctx,1),\n", + " leeched=leech_threshold_met(lapses, ctx.leech_threshold))\n", + " if ctx.relearn_steps:\n", + " again = RelearnSt(LearnSt(len(ctx.relearn_steps), step_again_delay(ctx.relearn_steps), mem=_mem(ctx,1)), again_review)\n", + " elif ctx.fsrs is not None and ctx.allow_short and (ctx.short_steps or not ctx.relearn_steps) and fail < 0.5:\n", + " again = RelearnSt(LearnSt(0, int(fail*86400), mem=_mem(ctx,1)), again_review)\n", + " else: again = again_review\n", + " return Answers(self, again, ReviewSt(hard, max(self.ease+EASE_HARD, MIN_EASE), self.lapses, mem=_mem(ctx,2)),\n", + " ReviewSt(good, self.ease, self.lapses, mem=_mem(ctx,3)),\n", + " ReviewSt(easy, self.ease+EASE_EASY, self.lapses, mem=_mem(ctx,4)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7371ea7", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "def _graduate(ctx, rating):\n", + " \"Leave the learning steps for review: `grad_good`/`grad_easy` days under SM-2, the model's interval under FSRS\"\n", + " lo,hi = _min_max(ctx, 1)\n", + " if ctx.fsrs is None:\n", + " ivl = ctx.grad_easy if rating==4 else ctx.grad_good\n", + " return ReviewSt(with_fuzz(ctx.fuzz, max(_round(ivl),1), lo, hi), ctx.init_ease)\n", + " st = ctx.fsrs[rating-1]\n", + " if rating==4: lo = with_fuzz(ctx.fuzz, ctx.fsrs[2].ivl, lo, hi) + 1 # Easy must clear the fuzzed Good interval\n", + " return ReviewSt(with_fuzz(ctx.fuzz, max(_round(st.ivl),1), lo, hi), ctx.init_ease, mem=st.mem)\n", + "\n", + "def _learn_short(ctx, rating, steps):\n", + " \"The FSRS short-term state when the model wants this answer back the same day, else None\"\n", + " if ctx.fsrs is None or not ctx.allow_short or not (ctx.short_steps or not steps): return None\n", + " st = ctx.fsrs[rating-1]\n", + " return st if st.ivl < 0.5 else None\n", + "\n", + "@patch\n", + "def next_answers(self:LearnSt, ctx):\n", + " steps = ctx.steps\n", + " def grad_or_short(rating, remaining):\n", + " s = _learn_short(ctx, rating, steps)\n", + " if s is not None: return LearnSt(remaining, int(s.ivl*86400), mem=s.mem)\n", + " return _graduate(ctx, rating)\n", + " ad = step_again_delay(steps)\n", + " again = LearnSt(len(steps), ad, mem=_mem(ctx,1)) if ad is not None else grad_or_short(1, len(steps))\n", + " hd = step_hard_delay(steps, self.remaining)\n", + " hard = LearnSt(self.remaining, hd, mem=_mem(ctx,2)) if hd is not None else grad_or_short(2, self.remaining)\n", + " gd = step_good_delay(steps, self.remaining)\n", + " good = LearnSt(step_remaining_good(steps, self.remaining), gd, mem=_mem(ctx,3)) if gd is not None else grad_or_short(3, self.remaining)\n", + " return Answers(self, again, hard, good, _graduate(ctx, 4))\n", + "\n", + "@patch\n", + "def next_answers(self:NewSt, ctx):\n", + " \"A new card answers like a learning card that just failed\"\n", + " return LearnSt(len(ctx.steps), 0).next_answers(ctx)._replace(current=self)\n", + "\n", + "def _relearn_pass(rl, ctx, rating):\n", + " \"A passing FSRS answer in relearning: back to review, or another same-day step if the model wants one\"\n", + " lo,hi = _min_max(ctx, 1)\n", + " st = ctx.fsrs[rating-1]\n", + " rev = rl.review._replace(ivl=with_fuzz(ctx.fuzz, max(_round(st.ivl),1), lo, hi), mem=st.mem)\n", + " if ctx.allow_short and (ctx.short_steps or not ctx.relearn_steps) and st.ivl < 0.5:\n", + " rem = rl.learn.remaining if rating==2 else step_remaining_good(ctx.relearn_steps, rl.learn.remaining)\n", + " return RelearnSt(rl.learn._replace(remaining=rem, secs=int(st.ivl*86400), elapsed=0, mem=st.mem), rev)\n", + " return rev\n", + "\n", + "@patch\n", + "def next_answers(self:RelearnSt, ctx):\n", + " steps,rev,fs = ctx.relearn_steps, self.review, ctx.fsrs\n", + " fail = rev._failing_ivl(ctx)\n", + " days = max(_round(max(fail,0)), 1)\n", + " ad = step_again_delay(steps)\n", + " if ad is not None: again = RelearnSt(LearnSt(len(steps), ad, mem=_mem(ctx,1)), rev._replace(ivl=days, elapsed=0, mem=_mem(ctx,1)))\n", + " elif fs is not None:\n", + " lo,hi = _min_max(ctx, 1)\n", + " again_rev = rev._replace(ivl=with_fuzz(ctx.fuzz, max(_round(fail),1), lo, hi), mem=fs[0].mem)\n", + " if ctx.allow_short and (ctx.short_steps or not steps) and fail < 0.5:\n", + " again = RelearnSt(LearnSt(len(steps), int(fail*86400), mem=fs[0].mem), again_rev)\n", + " else: again = again_rev\n", + " else: again = rev\n", + " hd = step_hard_delay(steps, self.learn.remaining)\n", + " if hd is not None: hard = RelearnSt(self.learn._replace(secs=hd, elapsed=0, mem=_mem(ctx,2)), rev._replace(elapsed=0, mem=_mem(ctx,2)))\n", + " elif fs is not None: hard = _relearn_pass(self, ctx, 2)\n", + " else: hard = rev\n", + " gd = step_good_delay(steps, self.learn.remaining)\n", + " if gd is not None:\n", + " good = RelearnSt(LearnSt(step_remaining_good(steps, self.learn.remaining), gd, mem=_mem(ctx,3)), rev._replace(elapsed=0, mem=_mem(ctx,3)))\n", + " elif fs is not None: good = _relearn_pass(self, ctx, 3)\n", + " else: good = rev\n", + " if fs is not None:\n", + " lo,hi = _min_max(ctx, 1)\n", + " lo = with_fuzz(ctx.fuzz, fs[2].ivl, lo, hi) + 1\n", + " easy = rev._replace(ivl=with_fuzz(ctx.fuzz, max(_round(fs[3].ivl),1), lo, hi), elapsed=0, mem=fs[3].mem)\n", + " else: easy = rev._replace(ivl=rev.ivl+1, elapsed=0)\n", + " return Answers(self, again, hard, good, easy)" + ] + }, + { + "cell_type": "markdown", + "id": "3841c1de", + "metadata": {}, + "source": [ + "The checks below are `review.rs`'s own unit tests: leech cadence at whole and half thresholds, the low-ease/low-multiplier interval ladder at fuzz 0 and 0.99, a silly 0.1x multiplier that must not underflow, and the maximum interval clamping everything to 5 days." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7865613", + "metadata": {}, + "outputs": [], + "source": [ + "for lapses,want in [(2,False),(3,True),(4,False),(5,True),(6,False),(7,True)]: test_eq(leech_threshold_met(lapses,3), want)\n", + "for lapses,want in [(7,False),(8,True),(9,False),(11,False),(12,True)]: test_eq(leech_threshold_met(lapses,8), want)\n", + "test_eq(leech_threshold_met(0,0), False)\n", + "for lapses,want in [(0,False),(1,True),(2,True),(3,True)]: test_eq(leech_threshold_met(lapses,1), want)\n", + "\n", + "_ctx = mk_ctx(cfg, fuzz=0.0)\n", + "st = ReviewSt(1, 1.3, elapsed=1)\n", + "test_eq(st._passing_ivls(_ctx), (2,3,4))\n", + "test_eq(st._passing_ivls(_ctx._replace(ivl_mult=0.1)), (2,3,4))\n", + "test_eq(st._passing_ivls(_ctx._replace(fuzz=0.99, ivl_mult=0.1)), (2,4,6))\n", + "test_eq(st._passing_ivls(_ctx._replace(fuzz=0.99, ivl_mult=10.0, max_ivl=5)), (5,5,5))\n", + "st2 = ReviewSt(2, 1.3, elapsed=2)\n", + "test_eq(st2._passing_ivls(_ctx._replace(hard_mult=0.1)), (1,3,4))" + ] + }, + { + "cell_type": "markdown", + "id": "0c727722", + "metadata": {}, + "source": [ + "And the shape of the whole machine on the default preset: a new card walks the 1m/10m steps, graduates to 1 day, and a lapsed review re-enters relearning at 10 minutes with its ease floored:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00389fcd", + "metadata": {}, + "outputs": [], + "source": [ + "_nf = mk_ctx(cfg) # fuzz=None: the deterministic path our oracle tests run under\n", + "a = NewSt(0).next_answers(_nf)\n", + "test_eq(a.again, LearnSt(2, 60))\n", + "test_eq(a.hard, LearnSt(2, 330))\n", + "test_eq(a.good, LearnSt(1, 600))\n", + "test_eq(a.easy, ReviewSt(4, 2.5))\n", + "test_eq(LearnSt(1, 600).next_answers(_nf).good, ReviewSt(1, 2.5)) # last step -> graduate\n", + "test_eq(LearnSt(1, 600).next_answers(_nf).again, LearnSt(2, 60)) # Again restarts the ladder\n", + "\n", + "r = ReviewSt(10, 2.5, elapsed=10).next_answers(_nf)\n", + "test_eq(r.again, RelearnSt(LearnSt(1, 600), ReviewSt(1, 2.3, 1)))\n", + "test_eq((r.hard, r.good, r.easy), (ReviewSt(12, 2.35), ReviewSt(25, 2.5), ReviewSt(32, 2.65)))\n", + "\n", + "rl = RelearnSt(LearnSt(1, 600), ReviewSt(1, 2.3, 1)).next_answers(_nf)\n", + "test_eq(rl.good, ReviewSt(1, 2.3, 1)) # relearning done: back to review\n", + "test_eq(rl.easy, ReviewSt(2, 2.3, 1))\n", + "test_eq(rl.again, RelearnSt(LearnSt(1, 600), ReviewSt(1, 2.3, 1)))" + ] + }, + { + "cell_type": "markdown", + "id": "ca103d4c", + "metadata": {}, + "source": [ + "Note the Easy interval: `10 * 2.5 * 1.3` looks like 32.5, but the preset's multipliers are protobuf *float32*s (`easy_multiplier` is really 1.2999999523...), so the product lands just under and rounds to 32 -- exactly as Anki's f32 arithmetic has it. Keeping the raw f32 values instead of \"tidying\" them is part of matching the oracle." + ] + }, + { + "cell_type": "markdown", + "id": "57368c97", + "metadata": {}, + "source": [ + "## Reading a card's state" + ] + }, + { + "cell_type": "markdown", + "id": "0a0bbaf2", + "metadata": {}, + "source": [ + "`card_state` recovers the state from a card row's scheduling columns, per `current.rs`. The columns pivot on `type` (0 new, 1 learning, 2 review, 3 relearning) while `queue` picks the due encoding: an intraday learning card (queue 1) stores a unix timestamp, a day-learner (queue 3) or review card a day number. A review card's `elapsed` days come from clamping `due` to today, so an overdue card gets credit for the wait; a learning card's elapsed seconds are reconstructed by re-deriving the fuzzed delay that was added when it was scheduled — recomputable because the fuzz is seeded by `(card, reps)`. Anki reconstructs with its own generator there, so across implementations that reconstruction can differ by the fuzz amount (25%/5min at most); it only feeds displays and FSRS's same-day accounting, never the stored schedule." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01548bec", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "def _card_data(c):\n", + " \"The card's `data` column as a dict\"\n", + " try: return json.loads(c.data) if c.data else {}\n", + " except ValueError: return {}\n", + "\n", + "def card_state(c, cfg, today, now=None):\n", + " \"The scheduling state of card row `c` under preset `cfg`\"\n", + " now = ifnone(now, int(time.time()))\n", + " left = c.left%1000\n", + " d = _card_data(c)\n", + " mem = MemSt(d['s'], d['d']) if 's' in d and 'd' in d else None\n", + " if c.type==0: return NewSt(max(c.due,0))\n", + " if c.type==2:\n", + " due = min(c.due, today)\n", + " return ReviewSt(c.ivl, c.factor/1000, c.lapses, elapsed=max(c.ivl-(due-today), 0), mem=mem)\n", + " steps = list(cfg.learn_steps if c.type==1 else cfg.relearn_steps)\n", + " last = step_current_delay(steps, left)\n", + " if c.queue==1: elapsed = now - (c.due - learn_fuzz(c.id+c.reps-1 if c.reps else None, last))\n", + " elif c.queue==3: elapsed = (today - c.due + max(last//DAY, 1))*DAY\n", + " else: elapsed = 0\n", + " learn = LearnSt(left, last, elapsed, mem=mem)\n", + " if c.type==1: return learn\n", + " return RelearnSt(learn, ReviewSt(c.ivl, c.factor/1000, c.lapses, elapsed=c.ivl, mem=mem))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a1b96cb", + "metadata": {}, + "outputs": [], + "source": [ + "_now = int(time.time())\n", + "crd = lambda **kw: Card(**{**dict(id=1, nid=1, did=1, ord=0, mod=0, usn=-1, type=0, queue=0, due=5, ivl=0), **kw})\n", + "test_eq(card_state(crd(), cfg, 10), NewSt(5))\n", + "test_eq(card_state(crd(type=2, queue=2, due=8, ivl=7, factor=2500), cfg, 10), ReviewSt(7, 2.5, elapsed=9))\n", + "test_eq(card_state(crd(type=2, queue=2, due=12, ivl=7, factor=2500), cfg, 10), ReviewSt(7, 2.5, elapsed=7)) # due is clamped to today: not-yet-due reads as on time\n", + "st = card_state(crd(type=1, queue=1, due=_now+300, ivl=0, left=1, reps=2), cfg, 10, now=_now)\n", + "test_eq((st.remaining, st.secs), (1, 600))\n", + "st = card_state(crd(type=3, queue=3, due=11, ivl=3, factor=2300, left=1, lapses=2), cfg, 10)\n", + "test_eq(st.review, ReviewSt(3, 2.3, 2, elapsed=3))\n", + "test_eq(st.learn.remaining, 1)" + ] + }, + { + "cell_type": "markdown", + "id": "cead562c", + "metadata": {}, + "source": [ + "## FSRS" + ] + }, + { + "cell_type": "markdown", + "id": "394bd681", + "metadata": {}, + "source": [ + "FSRS is a single global switch (the `fsrs` config key), so we always do what the user's other clients would. When it's on, the state machine's `ctx.fsrs` slot carries the four `(memory, interval)` outcomes from `fastanki.fsrs`, computed from the deck preset's parameters. Collections hold up to three parameter generations, and like Anki we prefer version 6, falling back to 5 then 4.5 (`preset_params`); desired retention can be overridden per deck; and whether FSRS may schedule *same-day* steps depends on the short-term parameters being non-zero (`allow_short_term`) plus a config flag for using them alongside explicit steps.\n", + "\n", + "A card that predates FSRS (or was answered by a pre-FSRS client) has no stored memory state, so like Anki we rebuild one by replaying its review log — `_fsrs_reviews` ports `reviews_for_fsrs`'s filtering (drop cramming and manual entries, restart at the last learning sequence or reset, day-granular elapsed times against the next rollover) and `_memory_from_revlog` follows `fsrs_item_for_memory_state`: a complete history replays from scratch, a truncated one starts from an SM-2 approximation of the first surviving entry." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "290dbd5f", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "RLog = namedtuple('RLog', 'id ease ivl lastIvl factor type')\n", + "\n", + "@patch\n", + "def fsrs_on(self:Collection):\n", + " \"Is FSRS enabled for this collection?\"\n", + " return bool(self.conf('fsrs', False))\n", + "\n", + "def preset_params(cfg):\n", + " \"The preset's raw FSRS parameters: version 6, else 5, else 4.5, else [] meaning the defaults\"\n", + " return list(cfg.fsrs_params_6 or cfg.fsrs_params_5 or cfg.fsrs_params_4)\n", + "\n", + "def allow_short_term(raw):\n", + " \"May FSRS schedule same-day steps? Requires non-zero short-term params (default params qualify)\"\n", + " if not raw: return True\n", + " return raw[17]>0 and raw[18]>0 if len(raw)>=19 else False\n", + "\n", + "def ignore_revlogs_before(cfg):\n", + " \"The preset's ignore-revlogs-before date as epoch ms (0 if unset)\"\n", + " s = cfg.ignore_revlogs_before_date\n", + " return int(datetime.strptime(s, '%Y-%m-%d').replace(tzinfo=timezone.utc).timestamp()*1000) if s else 0\n", + "\n", + "@patch\n", + "def _deck_dr(self:Collection, did, cfg):\n", + " \"Effective desired retention: the deck's own override, else the preset's\"\n", + " k = decks_pb2.Deck.KindContainer()\n", + " k.ParseFromString(self.q1('select kind from decks where id=?', did))\n", + " return k.normal.desired_retention if k.normal.HasField('desired_retention') else cfg.desired_retention" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6546e9b9", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "def _rl_days(e, next_day_at): return max(next_day_at - e.id//1000, 0)//86400\n", + "\n", + "def _fsrs_reviews(entries, next_day_at, ignore_before=0):\n", + " \"Filter a card's revlog rows for FSRS and compute (rating, delta_t) pairs; None if nothing usable\"\n", + " first_learn = first_grade = None\n", + " for i in reversed(range(len(entries))):\n", + " e = entries[i]\n", + " if e.type==3 and e.factor==0: continue # cramming\n", + " if e.ease>0 and e.id>ignore_before and (e.ivl>=1 or e.ivl<=-86400): first_grade = i\n", + " if e.ease>0 and e.type==0: first_learn = i\n", + " elif e.type==4 and e.factor==0: # reset\n", + " if first_learn is None and first_grade is None: return None\n", + " break\n", + " elif first_learn is not None: break\n", + " complete = first_learn is not None\n", + " if complete and entries[first_learn].id < ignore_before and first_learn < len(entries)-1:\n", + " complete,first_learn = False,None\n", + " start = first_learn if first_learn is not None else first_grade\n", + " if start is None: return None\n", + " kept = [e for e in entries[start:] if e.ease>0 and not (e.type==3 and e.factor==0)]\n", + " if not kept: return None\n", + " ds = [0] + [_rl_days(a, next_day_at)-_rl_days(b, next_day_at) for a,b in zip(kept, kept[1:])]\n", + " return [(e.ease, dt) for e,dt in zip(kept, ds)], complete, kept\n", + "\n", + "def _memory_from_revlog(w, entries, next_day_at, historical_retention=0.9, ignore_before=0):\n", + " \"Replay a card's revlog into a `MemSt`, starting a truncated history from an SM-2 approximation\"\n", + " out = _fsrs_reviews(entries, next_day_at, ignore_before)\n", + " if out is None: return None\n", + " revs,complete,kept = out\n", + " mem = None\n", + " if not complete:\n", + " first = kept[0]\n", + " ease = (first.factor or 2500)/1000\n", + " mem = memory_state_from_sm2(w, ease, max(first.ivl, 1), historical_retention)\n", + " if ease <= 1.1: mem = mem._replace(difficulty=f32((ease-0.1)*9 + 1)) # entry was written by FSRS itself\n", + " revs = revs[1:]\n", + " for rating,dt in revs: mem = step(w, dt, rating, mem)\n", + " return mem" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18d38eb4", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "@patch\n", + "def _fsrs_ctx(self:Collection, c, cfg, next_day):\n", + " \"(four FSRS next-states, desired retention, decay) for card `c`, rebuilding memory from the revlog when absent\"\n", + " raw = preset_params(cfg)\n", + " w = fsrs_params(raw)\n", + " d = _card_data(c)\n", + " mem = MemSt(d['s'], d['d']) if 's' in d and 'd' in d else None\n", + " if mem is None and c.type!=0:\n", + " rows = [RLog(*r) for r in self.q('select id, ease, ivl, lastIvl, factor, type from revlog where cid=? order by id', c.id)]\n", + " mem = _memory_from_revlog(w, rows, next_day, cfg.historical_retention, ignore_revlogs_before(cfg))\n", + " lrt = d.get('lrt') or self.q1('select max(id)/1000 from revlog where cid=? and ease between 1 and 4 and (type!=3 or factor!=0)', c.id)\n", + " days = max(next_day-lrt, 0)//86400 if lrt else 0\n", + " dr = self._deck_dr(c.did, cfg)\n", + " return next_states(w, mem, dr, days), dr, param_decay(raw)" + ] + }, + { + "cell_type": "markdown", + "id": "9be9b0c2", + "metadata": {}, + "source": [ + "The pure parts, on a synthetic history: a complete revlog replays to the same state as stepping by hand; a truncated one (no learning entries survive) starts from the SM-2 approximation; manual entries and cramming never count." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54f0f8b1", + "metadata": {}, + "outputs": [], + "source": [ + "wd = fsrs_params([])\n", + "_ms = lambda day: (1700000000+day*86400+3600)*1000 # an hour past the rollover, so same-day reviews share a day count\n", + "_nda = 1700000000+900*86400\n", + "rl = [RLog(_ms(0), 3, -600, 0, 0, 0), RLog(_ms(0)+600_000, 3, 1, -600, 2500, 0), RLog(_ms(1), 3, 3, 1, 2500, 1)]\n", + "byhand = step(wd, 1, 3, step(wd, 0, 3, step(wd, 0, 3, None)))\n", + "test_eq(_memory_from_revlog(wd, rl, _nda), byhand)\n", + "revs,complete,kept = _fsrs_reviews(rl, _nda)\n", + "test_eq((complete, [r for r,_ in revs], [dt for _,dt in revs]), (True, [3,3,3], [0,0,1]))\n", + "\n", + "trunc = [RLog(_ms(0), 3, 10, 5, 2300, 1), RLog(_ms(10), 4, 21, 10, 2450, 1)]\n", + "m = _memory_from_revlog(wd, trunc, _nda)\n", + "test_eq(m, step(wd, 10, 4, memory_state_from_sm2(wd, 2.3, 10)))\n", + "\n", + "test_is(_memory_from_revlog(wd, [RLog(_ms(0), 0, 5, 5, 0, 4)], _nda), None) # reset only: nothing usable\n", + "test_eq(_fsrs_reviews(rl+[RLog(_ms(2), 0, -1200, 3, 0, 3)], _nda)[0], revs) # cramming ignored\n", + "test_eq(allow_short_term([]), True)\n", + "test_eq(allow_short_term([0.1]*17), False)\n", + "test_eq(allow_short_term(list(DEFAULT_PARAMS)), True)" + ] + }, + { + "cell_type": "markdown", + "id": "080c3692", + "metadata": {}, + "source": [ + "## Answering" + ] + }, + { + "cell_type": "markdown", + "id": "9e62a457", + "metadata": {}, + "source": [ + "An answer leaves five marks, each following the sync bookkeeping rules from `fastanki.collection`:\n", + "- the card row is rewritten (with `usn=-1` and, for a card leaving the new queue, its original position tucked into the `data` JSON as `pos`, plus the answer time as `lrt`)\n", + "- a `revlog` row records the transition (this is the row other clients rebuild from)\n", + "- the deck and its parents bump their daily counters, which is how limits stay honest across devices mid-day\n", + "- a lapse at the leech threshold tags the note `leech` and optionally suspends the card\n", + "- sibling cards get buried per the preset.\n", + "\n", + "Filtered decks stay unsupported, as everywhere in fastanki.\n", + "\n", + "`fuzz=False` turns off both interval fuzz and the learning-delay fuzz, which is what `ANKI_TEST_MODE` does to the oracle — tests run both sides that way and compare exactly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb9dfbdd", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "@patch\n", + "def _card_answers(self:Collection, cid, fuzz, now):\n", + " \"Load card `cid` fresh and compute its state, the four answer outcomes, and FSRS extras\"\n", + " c = Card(*self.q('select * from cards where id=?', cid)[0])\n", + " assert not c.odid, \"cards in filtered decks are not supported\"\n", + " now = ifnone(now, int(time.time()))\n", + " today,next_day = self.timing()\n", + " cfg = self.deck_conf(c.did)\n", + " fs = dr = decay = None\n", + " if self.fsrs_on():\n", + " fs,dr,decay = self._fsrs_ctx(c, cfg, next_day)\n", + " raw = preset_params(cfg)\n", + " ctx = mk_ctx(cfg, fuzz=fuzz_factor(c.id, c.reps) if fuzz else None, fsrs=fs, allow_short=allow_short_term(raw),\n", + " short_steps=bool(self.conf('fsrsShortTermWithStepsEnabled', False)))\n", + " else: ctx = mk_ctx(cfg, fuzz=fuzz_factor(c.id, c.reps) if fuzz else None)\n", + " cur = card_state(c, cfg, today, now)\n", + " return c, cfg, cur, cur.next_answers(ctx), today, max(next_day-now, 0), now, dr, decay\n", + "\n", + "@patch\n", + "def answer_buttons(self:Collection, card, fuzz=True, now=None):\n", + " \"For each ease 1-4, `(next_state, delay_secs)` -- what a client shows on its answer buttons\"\n", + " _,_,_,ans,_,sur,_,_,_ = self._card_answers(card.id if isinstance(card,Card) else card, fuzz, now)\n", + " return [(st, ivl_secs(ivl_days(st.ivl_kind(), sur))) for st in ans[1:]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a13556f3", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "@patch\n", + "def _leech_note(self:Collection, nid, now):\n", + " tags = self.q1('select tags from notes where id=?', nid).split()\n", + " if 'leech' not in tags:\n", + " self.con.execute('update notes set tags=?, mod=?, usn=-1 where id=?', (f\" {' '.join(tags+['leech'])} \", now, nid))\n", + "\n", + "@patch\n", + "def _bump_deck_stats(self:Collection, did, today, new_delta, rev_delta, ms_delta):\n", + " \"Add an answer to the daily counters of deck `did` and its parents, resetting them on a new day\"\n", + " parts = self.q1('select name from decks where id=?', did).split('\\x1f')\n", + " names = ['\\x1f'.join(parts[:i]) for i in range(1, len(parts)+1)]\n", + " for did_,blob in self.q(f\"select id, common from decks where name in ({','.join('?'*len(names))})\", *names):\n", + " c = decks_pb2.Deck.Common()\n", + " c.ParseFromString(blob)\n", + " if c.last_day_studied != today:\n", + " c.new_studied,c.learning_studied,c.review_studied,c.milliseconds_studied = 0,0,0,0\n", + " c.last_day_studied = today\n", + " c.new_studied += new_delta\n", + " c.review_studied += rev_delta\n", + " c.milliseconds_studied += ms_delta\n", + " self.con.execute('update decks set common=?, mtime_secs=?, usn=-1 where id=?', (c.SerializeToString(), int(time.time()), did_))\n", + "\n", + "_GATHER_ORD = {1:0, 4:0, 3:1, 2:2, 0:3} # queue -> gather order: intraday learn, interday learn, review, new\n", + "\n", + "@patch\n", + "def _bury_siblings(self:Collection, c, cfg, now):\n", + " \"Bury (queue -2) siblings per the preset, only in queues gathered after the answered card's\"\n", + " g = _GATHER_ORD.get(c.queue, 99)\n", + " qs = [q for q,want in [(0,cfg.bury_new), (2,cfg.bury_reviews and g<=2), (3,cfg.bury_interday_learning and g<=1)] if want]\n", + " if qs: self.con.execute(f\"update cards set queue=-2, mod=?, usn=-1 where nid=? and id!=? and queue in ({','.join('?'*len(qs))})\",\n", + " (now, c.nid, c.id, *qs))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25b72dfe", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "def _shifted_d(mem):\n", + " \"FSRS difficulty normalized to the 0.1-1.1 range revlog factors use, x1000\"\n", + " return _round(((mem.difficulty-1)/9 + 0.1)*1000)\n", + "\n", + "@patch\n", + "def answer_card(self:Collection, card, ease, taken_ms=0, fuzz=True, now=None):\n", + " \"Answer `card` (a `Card` or id) with `ease` (1=Again 2=Hard 3=Good 4=Easy), returning the updated `Card`\"\n", + " cid = card.id if isinstance(card,Card) else card\n", + " assert ease in (1,2,3,4), f\"ease must be 1-4, got {ease}\"\n", + " with self._tx():\n", + " c,cfg,cur,ans,today,sur,now,dr,decay = self._card_answers(cid, fuzz, now)\n", + " nxt = ans[ease]\n", + " taken = min(taken_ms, cfg.cap_answer_time_to_secs*1000)\n", + " mem = nxt.learn.mem if isinstance(nxt, RelearnSt) else nxt.mem\n", + " d = _card_data(c)\n", + " if isinstance(cur, NewSt): d['pos'] = cur.position\n", + " d['lrt'] = now\n", + " for k in ('s','d','dr'): d.pop(k, None)\n", + " if mem is not None: d['s'],d['d'] = round(mem.stability, 4), round(mem.difficulty, 3)\n", + " if dr is not None: d['dr'] = round(dr, 2)\n", + " if decay is not None: d['decay'] = round(decay, 3)\n", + " cols = dict(mod=now, usn=-1, reps=c.reps+1, data=json.dumps(d, separators=(',',':')))\n", + " rl_fact = 0\n", + " if isinstance(nxt, ReviewSt):\n", + " cols.update(type=2, queue=2, ivl=nxt.ivl, due=today+nxt.ivl, factor=_round(nxt.ease*1000), lapses=nxt.lapses, left=0)\n", + " rl_fact = _shifted_d(mem) if mem is not None else cols['factor']\n", + " else:\n", + " learn = nxt if isinstance(nxt, LearnSt) else nxt.learn\n", + " if isinstance(nxt, LearnSt):\n", + " cols.update(type=1, left=learn.remaining)\n", + " rl_fact = _shifted_d(mem) if mem is not None else 0\n", + " else:\n", + " cols.update(type=3, left=learn.remaining, ivl=nxt.review.ivl, lapses=nxt.review.lapses, factor=_round(nxt.review.ease*1000))\n", + " rl_fact = _shifted_d(mem) if mem is not None else cols['factor']\n", + " iv = ivl_days(learn.ivl_kind(), sur)\n", + " if iv<0: cols.update(queue=1, due=now+learn_fuzz(c.id+c.reps if fuzz else None, -iv))\n", + " else: cols.update(queue=3, due=today+iv)\n", + " rev = nxt.review if isinstance(nxt, RelearnSt) else nxt\n", + " leeched = isinstance(rev, ReviewSt) and rev.leeched\n", + " if leeched and cfg.leech_action==0: cols['queue'] = -1 # LEECH_ACTION_SUSPEND\n", + " self.con.execute(f\"update cards set {', '.join(f'{k}=?' for k in cols)} where id=?\", (*cols.values(), cid))\n", + " self.con.execute('insert into revlog values (?,?,?,?,?,?,?,?,?)', (ts_id(self.con,'revlog'), cid, -1,\n", + " ease, ivl_days(nxt.ivl_kind(), sur), ivl_days(cur.ivl_kind(), sur), rl_fact, taken, cur.revlog_kind()))\n", + " self._bump_deck_stats(c.did, today, int(c.queue==0), int(c.queue in (2,3)), taken)\n", + " if leeched: self._leech_note(c.nid, now)\n", + " self._bury_siblings(c, cfg, now)\n", + " self._dirty()\n", + " return Card(*self.q('select * from cards where id=?', cid)[0])" + ] + }, + { + "cell_type": "markdown", + "id": "3f3d5d96", + "metadata": {}, + "source": [ + "A new card walked through its whole life, checking each mark an answer leaves. Good on a new card enters the second learning step; the revlog interval is `-600` (seconds, by the sign convention), the last interval 0, and the kind 0 (learning):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22d101f1", + "metadata": {}, + "outputs": [], + "source": [ + "n = col.add(Front='sched', Back='uler')\n", + "c = col.find_cards(Front='sched')[0]\n", + "c1 = col.answer_card(c, 3, taken_ms=1500, fuzz=False)\n", + "test_eq((c1.type, c1.queue, c1.left, c1.reps, c1.usn), (1, 1, 1, 1, -1))\n", + "assert abs(c1.due - (int(time.time())+600)) <= 3\n", + "test_eq(_card_data(c1), dict(pos=c.due, lrt=c1.mod))\n", + "rl = col.q('select ease, ivl, lastIvl, factor, time, type, usn from revlog where cid=?', c.id)\n", + "test_eq(rl, [(3, -600, 0, 0, 1500, 0, -1)])" + ] + }, + { + "cell_type": "markdown", + "id": "2319aabf", + "metadata": {}, + "source": [ + "Good again graduates it to a 1-day review card; the deck's daily counter has counted one new card; and failing the review tomorrow would drop it into relearning — shown here via `answer_buttons`, which is what a client renders (Again 10m, Hard ~12d, Good 25d... on a 10-day-old review below):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3dba794", + "metadata": {}, + "outputs": [], + "source": [ + "c2 = col.answer_card(c1, 3, fuzz=False)\n", + "test_eq((c2.type, c2.queue, c2.ivl, c2.factor, c2.left, c2.due), (2, 2, 1, 2500, 0, col.today()+1))\n", + "test_eq(col.q('select ivl, lastIvl, type from revlog where cid=? order by id', c.id)[-1], (1, -600, 0))\n", + "cmn = decks_pb2.Deck.Common()\n", + "cmn.ParseFromString(col.q1('select common from decks where id=1'))\n", + "test_eq((cmn.new_studied, cmn.last_day_studied, cmn.milliseconds_studied), (1, col.today(), 1500))\n", + "test_eq(col.q1('select usn from decks where id=1'), -1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c115084b", + "metadata": {}, + "outputs": [], + "source": [ + "col.con.execute('update cards set due=?, ivl=10 where id=?', (col.today(), c.id)) # a 10-day review, due today\n", + "btns = col.answer_buttons(c, fuzz=False)\n", + "test_eq([s for s,_ in btns], list(ReviewSt(10, 2.5, elapsed=10).next_answers(mk_ctx(cfg))[1:]))\n", + "test_eq(btns[0][1], 600) # Again: 10 minutes of relearning\n", + "test_eq(btns[2][1], 25*86400) # Good: 25 days\n", + "c3 = col.answer_card(c, 1, fuzz=False)\n", + "test_eq((c3.type, c3.queue, c3.ivl, c3.factor, c3.lapses, c3.left), (3, 1, 1, 2300, 1, 1))\n", + "test_eq(col.q('select ivl, lastIvl, factor, type from revlog where cid=? order by id', c.id)[-1], (-600, 10, 2300, 1))\n", + "c4 = col.answer_card(c3, 3, fuzz=False) # relearning done: back to review at the post-lapse interval\n", + "test_eq((c4.type, c4.queue, c4.ivl, c4.due), (2, 2, 1, col.today()+1))" + ] + }, + { + "cell_type": "markdown", + "id": "f81172b1", + "metadata": {}, + "source": [ + "## Choosing what to study" + ] + }, + { + "cell_type": "markdown", + "id": "b630f3e6", + "metadata": {}, + "source": [ + "`next_card` answers \"what now?\" for a study session: intraday learning cards whose delay has passed come first, then interday learning and reviews due today (both governed by the review limit), then new cards within the new-card limit, and finally (with nothing else to do), learning cards up to `collapseTime` early. Daily limits subtract the deck counters maintained by `answer_card`, so limits hold across devices and clients. Buried cards from previous days are restored first, the way Anki does when building its queues (deliberately without marking them modified, matching `unbury_on_day_rollover`).\n", + "\n", + "This is a deliberately simplified port of Anki's v3 queue builder: limits come from the *named deck's* preset rather than a per-subdeck limit tree, gathering is by due order (matching Anki's default \"Deck\" gather with sequential positions), reviews always precede new cards, and there is no display-order matrix. Those affect session ordering, not scheduling state, so they can grow later without compatibility concerns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12becb8a", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "@patch\n", + "def unbury_if_day_changed(self:Collection):\n", + " \"Restore buried cards once the day has rolled over since the last unbury\"\n", + " today,last = self.today(), self.conf('lastUnburied', 0)\n", + " if last < today or today+7 < last:\n", + " for cid,typ,due in self.q('select id, type, due from cards where queue in (-2,-3)'):\n", + " q = {0:0, 2:2}.get(typ, 1 if due>1_000_000_000 else 3)\n", + " self.con.execute('update cards set queue=? where id=?', (q, cid))\n", + " self.con.execute(\"insert or replace into config values ('lastUnburied',-1,?,?)\",\n", + " (now_ms()//1000, json.dumps(today).encode()))\n", + "\n", + "@patch\n", + "def _day_limits(self:Collection, did, today):\n", + " \"(new_left, review_left) for deck `did` today\"\n", + " cfg = self.deck_conf(did)\n", + " cmn = decks_pb2.Deck.Common()\n", + " cmn.ParseFromString(self.q1('select common from decks where id=?', did))\n", + " new_done,rev_done = (cmn.new_studied,cmn.review_studied) if cmn.last_day_studied==today else (0,0)\n", + " new_left,rev_left = max(cfg.new_per_day-new_done, 0), max(cfg.reviews_per_day-rev_done, 0)\n", + " if not self.conf('newCardsIgnoreReviewLimit', False): new_left = min(new_left, rev_left)\n", + " return new_left, rev_left\n", + "\n", + "@patch\n", + "def next_card(self:Collection, deck=None, fuzz=True):\n", + " \"The next card to study in `deck` (default the whole collection, limits from 'Default'), or None\"\n", + " self.unbury_if_day_changed()\n", + " today,now = self.today(), int(time.time())\n", + " new_left,rev_left = self._day_limits(self.deck_id(deck) if deck else 1, today)\n", + " cond,ps = self._find_sql(deck)\n", + " def pick(extra, *xps):\n", + " r = self.q(f'select c.* from cards c join notes n on c.nid=n.id where {cond} and {extra} order by c.due, c.id limit 1', *ps, *xps)\n", + " return Card(*r[0]) if r else None\n", + " c = pick('c.queue=1 and c.due<=?', now)\n", + " if not c and rev_left: c = pick('c.queue=3 and c.due<=?', today)\n", + " if not c and rev_left: c = pick('c.queue=2 and c.due<=?', today)\n", + " if not c and new_left: c = pick('c.queue=0')\n", + " if not c: c = pick('c.queue=1 and c.due<=?', now+self.conf('collapseTime',1200))\n", + " return c" + ] + }, + { + "cell_type": "markdown", + "id": "c43df409", + "metadata": {}, + "source": [ + "The collection currently holds yesterday's graduate (due tomorrow) and the untouched hola/uno cards from earlier notebooks' pattern — here just the one relearning graduate plus the new cloze pair. A fresh session serves the new cards in position order, and burying kicks in when enabled: answering one cloze sibling hides the other until tomorrow, and `unbury_if_day_changed` brings it back when the day counter moves:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4dc93f5", + "metadata": {}, + "outputs": [], + "source": [ + "cz = col.add(model='Cloze', Text='{{c1::sib}} {{c2::ling}}')\n", + "nxt = col.next_card()\n", + "test_eq((nxt.nid, nxt.ord), (cz.id, 0)) # reviews done for today: first new card, by position\n", + "\n", + "col.con.execute('update deck_config set config=? where id=1',\n", + " (col.deck_conf(1).__class__(bury_new=True).SerializeToString(),)) # minimal preset: bury_new on\n", + "cfg1 = col.deck_conf(1)\n", + "assert cfg1.bury_new\n", + "col.answer_card(nxt, 3, fuzz=False)\n", + "sib = col.q('select queue from cards where nid=? and ord=1', cz.id)[0][0]\n", + "test_eq(sib, -2) # sibling buried for the rest of today\n", + "test_is(col.next_card() is None or col.next_card().nid != cz.id, True)\n", + "\n", + "col.con.execute(\"insert or replace into config values ('lastUnburied',-1,0,?)\", (json.dumps(col.today()-1).encode(),))\n", + "col.unbury_if_day_changed()\n", + "test_eq(col.q('select queue from cards where nid=? and ord=1', cz.id)[0][0], 0)" + ] + }, + { + "cell_type": "markdown", + "id": "8f21a95a", + "metadata": {}, + "source": [ + "## The oracle test" + ] + }, + { + "cell_type": "markdown", + "id": "46b93008", + "metadata": {}, + "source": [ + "A collection built by fastanki is copied byte-for-byte, one copy driven by us and the other by Anki, so every id matches and rows compare directly. Before each answer we check **prediction parity**: our four `next_answers` against the oracle's `get_scheduling_states` (elapsed seconds excluded, as Anki itself does when comparing states; ease factors compared at Anki's own stored precision). After each answer, **application parity**: the entire scheduling row, the revlog, the `data` JSON and the deck counters must match, with a few seconds' tolerance only where wall-clock timestamps enter (an intraday due date, the `lrt` stamp).\n", + "\n", + "The script walks every SM-2 path: a new card through both learning steps (with a Hard on the first step to hit the averaging quirk), graduation, passing reviews at each button, a lapse into relearning and out again, Easy graduation straight from learning, and — with the leech threshold dropped to 1 — a lapse that tags the note `leech` on both sides." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c5dc8ed", + "metadata": {}, + "outputs": [], + "source": [ + "def _canon(st):\n", + " \"A state as comparable atoms: ease at Anki's stored precision, elapsed dropped\"\n", + " if isinstance(st, NewSt): return ('new', st.position)\n", + " if isinstance(st, LearnSt): return ('learn', st.remaining, st.secs)\n", + " if isinstance(st, ReviewSt): return ('review', st.ivl, _round(st.ease*1000), st.lapses, st.leeched)\n", + " return ('relearn', _canon(st.learn), _canon(st.review))\n", + "\n", + "def _st_of(p):\n", + " \"Our state for one of the oracle's `SchedulingState` protos\"\n", + " n = p.normal\n", + " k = n.WhichOneof('kind')\n", + " if k=='new': return NewSt(n.new.position)\n", + " if k=='learning': return LearnSt(n.learning.remaining_steps, n.learning.scheduled_secs)\n", + " if k=='review': return ReviewSt(n.review.scheduled_days, n.review.ease_factor, n.review.lapses, leeched=n.review.leeched)\n", + " r,l = n.relearning.review, n.relearning.learning\n", + " return RelearnSt(LearnSt(l.remaining_steps, l.scheduled_secs), ReviewSt(r.scheduled_days, r.ease_factor, r.lapses, leeched=r.leeched))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7a49cf1", + "metadata": {}, + "outputs": [], + "source": [ + "_RATING = {1:CardAnswer.AGAIN, 2:CardAnswer.HARD, 3:CardAnswer.GOOD, 4:CardAnswer.EASY}\n", + "\n", + "def _oracle_answer(oc, cid, ease):\n", + " st = oc._backend.get_scheduling_states(cid)\n", + " new_state = [None, st.again, st.hard, st.good, st.easy][ease]\n", + " oc.sched.answer_card(CardAnswer(card_id=cid, current_state=st.current, new_state=new_state,\n", + " rating=_RATING[ease], answered_at_millis=int(time.time()*1000), milliseconds_taken=1500))\n", + "\n", + "def _cmp_card(mycol, oc, cid):\n", + " cols = 'type, queue, ivl, factor, reps, lapses, left, due, data'\n", + " a = mycol.q(f'select {cols} from cards where id=?', cid)[0]\n", + " b = oc.db.execute(f'select {cols} from cards where id=?', cid)[0]\n", + " test_eq(a[:7], tuple(b[:7]))\n", + " if a[1]==1: assert abs(a[7]-b[7])<=5, f'learning due: {a[7]} vs {b[7]}'\n", + " else: test_eq(a[7], b[7])\n", + " da,db_ = json.loads(a[8] or '{}'), json.loads(b[8] or '{}')\n", + " assert abs(da.pop('lrt',0)-db_.pop('lrt',0))<=5\n", + " for k,tol in [('s',0.02),('d',0.02),('dr',1e-6),('decay',1e-6)]: # float32 vs float64 arithmetic: compare close\n", + " assert abs(da.pop(k,0)-db_.pop(k,0))<=tol, f'{k}: {da} vs {db_}'\n", + " test_eq(da, db_)\n", + " ra = mycol.q('select ease, ivl, lastIvl, factor, type, usn from revlog where cid=? order by id', cid)\n", + " rb = oc.db.execute('select ease, ivl, lastIvl, factor, type, usn from revlog where cid=? order by id', cid)\n", + " test_eq(ra, [tuple(r) for r in rb])" + ] + }, + { + "cell_type": "markdown", + "id": "538691ca", + "metadata": {}, + "source": [ + "Build the collection, twin it, and walk the script. Every step asserts prediction parity across all four buttons before answering, then answers on both sides and asserts application parity:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fdaf07c6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "prediction and application parity across 12 answers\n" + ] + } + ], + "source": [ + "otd = Path(tempfile.mkdtemp())\n", + "mycol = Collection.open(otd/'mine'/'collection.anki2')\n", + "n1 = mycol.add(Front='oracle', Back='test')\n", + "n2 = mycol.add(model='Cloze', Text='{{c1::twin}} {{c2::files}}')\n", + "cid1 = mycol.find_card_ids(where='n.id=?', args=[n1.id])[0]\n", + "cid2,cid3 = mycol.find_card_ids(where='n.id=?', args=[n2.id])\n", + "mycol._clear_wal()\n", + "shutil.copy(mycol.path, otd/'theirs.anki2')\n", + "oc = AnkiCollection(str(otd/'theirs.anki2'))\n", + "\n", + "def parity_step(mycol, oc, cid, ease):\n", + " \"Assert prediction parity on all four buttons, answer on both sides, assert application parity\"\n", + " ost = oc._backend.get_scheduling_states(cid)\n", + " _,_,cur,ans,_,_,_,_,_ = mycol._card_answers(cid, False, None)\n", + " for name,op in [('current',ost.current),('again',ost.again),('hard',ost.hard),('good',ost.good),('easy',ost.easy)]:\n", + " test_eq(_canon(getattr(ans, name)), _canon(_st_of(op)))\n", + " mycol.answer_card(cid, ease, taken_ms=1500, fuzz=False)\n", + " _oracle_answer(oc, cid, ease)\n", + " _cmp_card(mycol, oc, cid)\n", + "\n", + "for ease in (3, 2, 3, 3, 1, 3, 4): parity_step(mycol, oc, cid1, ease) # steps, graduate, review Good, lapse, relearn out, Easy\n", + "for ease in (1, 4, 3): parity_step(mycol, oc, cid2, ease) # Again on new, Easy straight out of learning, review\n", + "for ease in (3, 3): parity_step(mycol, oc, cid3, ease) # the sibling walks the plain path\n", + "print('prediction and application parity across', 7+3+2, 'answers')" + ] + }, + { + "cell_type": "markdown", + "id": "396d58f6", + "metadata": {}, + "source": [ + "Deck counters must agree too — and the leech path: drop the threshold to 1 on both sides, lapse a review card, and both implementations must tag the note and record the same lapse:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53dbdf4e", + "metadata": {}, + "outputs": [], + "source": [ + "cmn_a = decks_pb2.Deck.Common()\n", + "cmn_a.ParseFromString(mycol.q1('select common from decks where id=1'))\n", + "cmn_b = decks_pb2.Deck.Common()\n", + "cmn_b.ParseFromString(bytes(oc.db.execute('select common from decks where id=1')[0][0]))\n", + "for f in ('new_studied','review_studied','last_day_studied'): test_eq(getattr(cmn_a,f), getattr(cmn_b,f))\n", + "\n", + "lcfg = mycol.deck_conf(1)\n", + "lcfg.leech_threshold = 1\n", + "mycol.con.execute('update deck_config set config=? where id=1', (lcfg.SerializeToString(),))\n", + "oc.close()\n", + "ocon = connect(otd/'theirs.anki2')\n", + "ocon.execute('update deck_config set config=? where id=1', (lcfg.SerializeToString(),))\n", + "ocon.close()\n", + "oc = AnkiCollection(str(otd/'theirs.anki2'))\n", + "parity_step(mycol, oc, cid1, 1)\n", + "test_eq(mycol.get_note(n1.id).tags, ['leech'])\n", + "test_eq(oc.get_note(n1.id).tags, ['leech'])\n", + "oc.close()\n", + "mycol.close()" + ] + }, + { + "cell_type": "markdown", + "id": "68c3b7c0", + "metadata": {}, + "source": [ + "And the same but with **FSRS enabled** — including the reconstruction path. The twins first answer one card twice under SM-2, then the `fsrs` config flag is switched on both sides, so that card's memory state must be rebuilt from its revlog while fresh cards walk the learning steps with model-driven graduation. Stored stability/difficulty/retention/decay in the `data` column must match the oracle's at every step:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6d3fc06", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FSRS parity, including revlog reconstruction\n" + ] + } + ], + "source": [ + "ftd = Path(tempfile.mkdtemp())\n", + "fcol = Collection.open(ftd/'mine'/'collection.anki2')\n", + "fn1 = fcol.add(Front='fsrs', Back='oracle')\n", + "fn2 = fcol.add(model='Cloze', Text='{{c1::mem}} {{c2::state}}')\n", + "fc1 = fcol.find_card_ids(where='n.id=?', args=[fn1.id])[0]\n", + "fc2,fc3 = fcol.find_card_ids(where='n.id=?', args=[fn2.id])\n", + "fcol._clear_wal()\n", + "shutil.copy(fcol.path, ftd/'theirs.anki2')\n", + "foc = AnkiCollection(str(ftd/'theirs.anki2'))\n", + "for ease in (3, 3): parity_step(fcol, foc, fc3, ease) # SM-2 history first: revlog for the reconstruction path\n", + "\n", + "foc.close()\n", + "fcon = connect(ftd/'theirs.anki2')\n", + "for con in (fcol.con, fcon): con.execute(\"insert or replace into config values ('fsrs',-1,0,?)\", (json.dumps(True).encode(),))\n", + "fcon.close()\n", + "foc = AnkiCollection(str(ftd/'theirs.anki2'))\n", + "\n", + "for ease in (3, 3, 1, 3): parity_step(fcol, foc, fc1, ease) # steps and model graduation, lapse, relearn out\n", + "for ease in (4, 2): parity_step(fcol, foc, fc2, ease) # Easy from new, then Hard on the young review\n", + "parity_step(fcol, foc, fc3, 3) # memory state rebuilt from the SM-2 revlog\n", + "sd = _card_data(Card(*fcol.q('select * from cards where id=?', fc3)[0]))\n", + "assert 's' in sd and 'd' in sd and sd['dr']==0.9\n", + "foc.close()\n", + "fcol.close()\n", + "print('FSRS parity, including revlog reconstruction')" + ] + } + ], + "metadata": { + "solveit": { + "default_code": true, + "mode": "learning", + "use_thinking": true, + "use_tools": true, + "ver": 2 + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/03_core.ipynb b/nbs/05_core.ipynb similarity index 58% rename from nbs/03_core.ipynb rename to nbs/05_core.ipynb index 86f25fd..52a3b18 100644 --- a/nbs/03_core.ipynb +++ b/nbs/05_core.ipynb @@ -42,7 +42,8 @@ "from fastcore.utils import *\n", "from fastanki.schema import *\n", "from fastanki.collection import *\n", - "from fastanki.syncer import *" + "from fastanki.syncer import *\n", + "from fastanki.scheduler import *" ] }, { @@ -131,7 +132,7 @@ { "data": { "text/plain": [ - "1783999334909" + "1784903934511" ] }, "execution_count": null, @@ -252,7 +253,7 @@ "" ], "text/plain": [ - "Note(1783999334908, Front='What is the capital of France?', Back='Paris', tags=['geo'])" + "Note(1784903934509, Front='What is the capital of France?', Back='Paris', tags=['geo'])" ] }, "execution_count": null, @@ -320,6 +321,299 @@ "test_eq(find_note_ids(tag='geo'), [])" ] }, + { + "cell_type": "markdown", + "id": "46ec562e", + "metadata": {}, + "source": [ + "## Reviewing" + ] + }, + { + "cell_type": "markdown", + "id": "0f843aaf", + "metadata": {}, + "source": [ + "The study loop a client drives: `next_card` picks what to show, `answer_buttons` says what each button would schedule (for display), `answer_card` grades it. Scheduling follows the collection's own settings — SM-2 or FSRS, whichever the user's other clients use — and every answer lands in the revlog, so a later `sync` carries it to AnkiWeb and on to every other device." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3889fa2", + "metadata": { + "nbdev": { + "export": "true" + } + }, + "outputs": [], + "source": [ + "def next_card(\n", + " deck:str=None, # Deck name (subdecks included; default: whole collection)\n", + "):\n", + " \"The next card due for study, or None when the session is done.\"\n", + " with Collection.open() as col: return col.next_card(deck)\n", + "\n", + "def answer_buttons(\n", + " card_id:int, # Id of the card being reviewed\n", + "):\n", + " \"For each ease 1-4, `(next_state, delay_secs)`: what to show on the answer buttons.\"\n", + " with Collection.open() as col: return col.answer_buttons(card_id)\n", + "\n", + "def answer_card(\n", + " card_id:int, # Id of the card being reviewed\n", + " ease:int, # 1=Again 2=Hard 3=Good 4=Easy\n", + " taken_ms:int=0, # Milliseconds spent answering, for the stats\n", + "):\n", + " \"Answer a due card, updating its schedule and review log.\"\n", + " with Collection.open() as col: return col.answer_card(card_id, ease, taken_ms=taken_ms)\n", + "\n", + "def due_counts(\n", + " deck:str=None, # Deck name (subdecks included; default: whole collection)\n", + "):\n", + " \"(new, learning, review) counts due now.\"\n", + " with Collection.open() as col: return col.due_counts(deck)" + ] + }, + { + "cell_type": "markdown", + "id": "8d943cfe", + "metadata": {}, + "source": [ + "Let's demo studying. We initialise three new cards in their own deck, and conduct a session exactly like the one Anki's clients run: keep asking `next_card` until nothing is due.\n", + "\n", + "`answer_buttons` reports what each ease would schedule, in seconds — the numbers a client renders above its buttons — so a two-line formatter turns a card into the reviewer's view of it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ef7e9f3f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'perro [Again 1m · Hard 6m · Good 10m · Easy 3d]'" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "for f,b in [('perro','dog'),('gato','cat'),('pájaro','bird')]: add_fb_card(f, b, deck='Práctica')\n", + "\n", + "def delay(secs):\n", + " for n,u in [(86400,'d'),(3600,'h'),(60,'m')]:\n", + " if secs>=n: return f'{round(secs/n)}{u}'\n", + " return f'{secs}s'\n", + "\n", + "def show(c):\n", + " btns = ' · '.join(f'{l} {delay(s)}' for l,(_,s) in zip(('Again','Hard','Good','Easy'), answer_buttons(c.id)))\n", + " return f\"{get_note(c.nid)['Front']:8} [{btns}]\"\n", + "\n", + "show(next_card('Práctica'))" + ] + }, + { + "cell_type": "markdown", + "id": "c91052b8", + "metadata": {}, + "source": [ + "Those are the default learning steps: Again restarts at 1 minute, Good advances to the 10-minute step, Easy skips straight to a 4-day review. If we answer Good all the way through, the session settles in one sitting — when nothing else is waiting, Anki serves learning cards up to 20 minutes early, so each card appears twice (the 1-minute step, then the 10-minute step) and graduates to tomorrow:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "489444de", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perro [Again 1m · Hard 6m · Good 10m · Easy 3d]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gato [Again 1m · Hard 6m · Good 10m · Easy 3d]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "pájaro [Again 1m · Hard 6m · Good 10m · Easy 3d]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gato [Again 1m · Hard 10m · Good 1d · Easy 3d]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perro [Again 1m · Hard 10m · Good 1d · Easy 3d]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "pájaro [Again 1m · Hard 10m · Good 1d · Easy 5d]\n" + ] + }, + { + "data": { + "text/plain": [ + "(0, 0, 0)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "while (c := next_card('Práctica')):\n", + " print(show(c))\n", + " answer_card(c.id, 3)\n", + "due_counts('Práctica')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c476d08", + "metadata": {}, + "outputs": [], + "source": [ + "c = find_cards(deck='Práctica')[0]\n", + "test_eq((c.type, c.queue, c.ivl), (2, 2, 1)) # graduated: review cards on a 1-day interval" + ] + }, + { + "cell_type": "markdown", + "id": "5761b54f", + "metadata": {}, + "source": [ + "Let's fast-forward: make `perro` a 7-day review card due today. Now the buttons show SM-2's ladder — Hard grows the interval a little and eases off, Good multiplies by the card's ease factor, Easy adds a bonus — and failing it drops the card into relearning, with a lapse on its record and its ease knocked down:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ea9266d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perro [Again 10m · Hard 8d · Good 16d · Easy 21d]\n" + ] + }, + { + "data": { + "text/plain": [ + "(3, 1, 1, 2300)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with Collection.open() as col: col.con.execute('update cards set ivl=7, due=? where id=?', (col.today(), c.id))\n", + "print(show(c))\n", + "lapsed = answer_card(c.id, 1)\n", + "(lapsed.type, lapsed.queue, lapsed.lapses, lapsed.factor)" + ] + }, + { + "cell_type": "markdown", + "id": "4e337dfa", + "metadata": {}, + "source": [ + "Every answer lands in the review log: the button pressed, the interval it produced (positive days, negative seconds), the one it replaced, and the kind of review. This is the durable record — `sync` carries it to AnkiWeb, and any other client can rebuild scheduling state from it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b73df721", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(3, -600, 0, 0), (3, 1, -600, 0), (1, -600, 7, 1)]" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with Collection.open() as col: rlog = col.q('select ease, ivl, lastIvl, type from revlog where cid=?', c.id)\n", + "rlog" + ] + }, + { + "cell_type": "markdown", + "id": "8c5305dd", + "metadata": {}, + "source": [ + "Scheduling follows the collection's own configuration. Enable FSRS (the flag your other clients set when you turn it on in deck options) and the very same loop schedules with the memory model from `fastanki.fsrs` instead, no code changes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86a22648", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'perro [Again 10m · Hard 15m · Good 1d · Easy 2d]'" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with Collection.open() as col: col.con.execute(\"insert or replace into config values ('fsrs',-1,0,?)\", (b'true',))\n", + "show(c)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77d9c2d7", + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", + "with Collection.open() as col:\n", + " col.con.execute(\"delete from config where key='fsrs'\")\n", + " col.remove_deck('Práctica')" + ] + }, { "cell_type": "markdown", "id": "d8459c43", @@ -388,7 +682,7 @@ }, "outputs": [], "source": [ - "def anki_tools(): print('&`[add_card, add_fb_card, add_cloze_card, find_notes, find_note_ids, find_cards, find_card_ids, get_note, del_note, update_fb_note, sync]`')" + "def anki_tools(): print('&`[add_card, add_fb_card, add_cloze_card, find_notes, find_note_ids, find_cards, find_card_ids, get_note, del_note, update_fb_note, next_card, answer_buttons, answer_card, due_counts, sync]`')" ] }, { @@ -398,24 +692,11 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "['add_card',\n", - " 'add_fb_card',\n", - " 'add_cloze_card',\n", - " 'find_notes',\n", - " 'find_note_ids',\n", - " 'find_cards',\n", - " 'find_card_ids',\n", - " 'get_note',\n", - " 'del_note',\n", - " 'update_fb_note',\n", - " 'sync']" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "&`[add_card, add_fb_card, add_cloze_card, find_notes, find_note_ids, find_cards, find_card_ids, get_note, del_note, update_fb_note, next_card, answer_buttons, answer_card, due_counts, sync]`\n" + ] } ], "source": [ diff --git a/nbs/index.ipynb b/nbs/index.ipynb index 859154d..cfee670 100644 --- a/nbs/index.ipynb +++ b/nbs/index.ipynb @@ -129,7 +129,7 @@ { "data": { "text/plain": [ - "[Card(1764738198390, nid=1764738198390, due=49, ivl=0, queue=0)]" + "[Card(1784904710443, nid=1784904710443, due=1, ivl=0, queue=0)]" ] }, "execution_count": null, @@ -151,10 +151,14 @@ { "data": { "text/markdown": [ - "Card 1764738198390 (nid: 1764738198390, due: 49, ivl: 0d, queue: 0)" + "
\n", + "\n", + "Card 1784904710443 (nid: 1784904710443): new #1\n", + "\n", + "
" ], "text/plain": [ - "Card(1764738198390, nid=1764738198390, due=49, ivl=0, queue=0)" + "Card(1784904710443, nid=1784904710443, due=1, ivl=0, queue=0)" ] }, "execution_count": null, @@ -175,7 +179,7 @@ { "data": { "text/plain": [ - "[1764738198390]" + "[1784904710443]" ] }, "execution_count": null, @@ -204,7 +208,7 @@ { "data": { "text/plain": [ - "[Note(1764738198390, Front='你好', Back='hello', tags=[])]" + "[Note(1784904710443, Front='你好', Back='hello', tags=[])]" ] }, "execution_count": null, @@ -226,10 +230,14 @@ { "data": { "text/markdown": [ - "**Front**: 你好 | **Back**: hello" + "
\n", + "\n", + "**Front**: 你好 | **Back**: hello | 🏷 -\n", + "\n", + "
" ], "text/plain": [ - "Note(1764738198390, Front='你好', Back='hello', tags=[])" + "Note(1784904710443, Front='你好', Back='hello', tags=[])" ] }, "execution_count": null, @@ -251,7 +259,7 @@ { "data": { "text/plain": [ - "[1764738198390]" + "[1784904710443]" ] }, "execution_count": null, @@ -282,10 +290,14 @@ { "data": { "text/markdown": [ - "**Front**: 你好 | **Back**: updated answer | 🏷️ testtag" + "
\n", + "\n", + "**Front**: 你好 | **Back**: updated answer | 🏷 testtag\n", + "\n", + "
" ], "text/plain": [ - "Note(1764738198390, Front='你好', Back='updated answer', tags=['testtag'])" + "Note(1784904710443, Front='你好', Back='updated answer', tags=['testtag'])" ] }, "execution_count": null, @@ -306,10 +318,14 @@ { "data": { "text/markdown": [ - "**Front**: 你好 | **Back**: updated answer | 🏷️ testtag, moretagz" + "
\n", + "\n", + "**Front**: 你好 | **Back**: updated answer | 🏷 testtag moretagz\n", + "\n", + "
" ], "text/plain": [ - "Note(1764738198390, Front='你好', Back='updated answer', tags=['testtag', 'moretagz'])" + "Note(1784904710443, Front='你好', Back='updated answer', tags=['testtag', 'moretagz'])" ] }, "execution_count": null, @@ -330,10 +346,14 @@ { "data": { "text/markdown": [ - "**Front**: 你好 | **Back**: updated answer | 🏷️ moretagz, testtag" + "
\n", + "\n", + "**Front**: 你好 | **Back**: updated answer | 🏷 testtag moretagz\n", + "\n", + "
" ], "text/plain": [ - "Note(1764738198390, Front='你好', Back='updated answer', tags=['moretagz', 'testtag'])" + "Note(1784904710443, Front='你好', Back='updated answer', tags=['testtag', 'moretagz'])" ] }, "execution_count": null, @@ -353,11 +373,8 @@ "outputs": [ { "data": { - "text/markdown": [ - "✓ 1 change(s)" - ], "text/plain": [ - "OpChangesWithCount(1)" + "2" ] }, "execution_count": null, @@ -371,22 +388,59 @@ }, { "cell_type": "markdown", - "id": "15712e06", + "id": "a00cca6c", "metadata": {}, "source": [ - "`sync` connects to AnkiWeb: pass your credentials the first time, and they're saved (as a host key, not your password) for later calls. The first sync of a fresh collection is a full download of your existing AnkiWeb collection; after that, syncs exchange deltas in both directions. fastanki will never replace a non-empty server collection without an explicit `upload=True`." + "Reviewing is the same loop every Anki client runs: `next_card` picks what to study, `answer_buttons` reports each button's delay (for display), `answer_card` grades it (1=Again, 2=Hard, 3=Good, 4=Easy). New cards walk the learning steps and graduate to review cards; scheduling follows the collection's own settings (SM-2 or FSRS, whichever your other clients use) and every answer is a review-log entry that `sync` carries to all your devices. A fresh card settles in one sitting, because learning cards are served a little early once nothing else is due:" ] }, { "cell_type": "code", "execution_count": null, - "id": "ac69cb7c", + "id": "7b5c06a0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "¿cómo estás? [Again 1m · Hard 6m · Good 10m · Easy 4d]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "¿cómo estás? [Again 1m · Hard 10m · Good 1d · Easy 4d]\n" + ] + } + ], + "source": [ + "nid = add_fb_card('¿cómo estás?', 'how are you?')\n", + "def delay(s): return f'{round(s/86400)}d' if s>=86400 else f'{round(s/60)}m'\n", + "while (card := next_card()):\n", + " labels = ' · '.join(f'{l} {delay(s)}' for l,(_,s) in zip(('Again','Hard','Good','Easy'), answer_buttons(card.id)))\n", + " print(f\"{get_note(card.nid)['Front']} [{labels}]\")\n", + " answer_card(card.id, 3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc7c69d9", "metadata": {}, "outputs": [ { "data": { + "text/markdown": [ + "
\n", + "\n", + "Card 1784904710676 (nid: 1784904710676): review, ivl 1d, due day 1\n", + "\n", + "
" + ], "text/plain": [ - "host_number: 5" + "Card(1784904710676, nid=1784904710676, due=1, ivl=1, queue=2)" ] }, "execution_count": null, @@ -394,6 +448,69 @@ "output_type": "execute_result" } ], + "source": [ + "find_cards(fields={'Front':'cómo'})[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9dfabdcf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#| hide\n", + "del_note(nid)" + ] + }, + { + "cell_type": "markdown", + "id": "15712e06", + "metadata": {}, + "source": [ + "`sync` connects to AnkiWeb: pass your credentials the first time, and they're saved (as a host key, not your password) for later calls. The first sync of a fresh collection is a full download of your existing AnkiWeb collection; after that, syncs exchange deltas in both directions. fastanki will never replace a non-empty server collection without an explicit `upload=True`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac69cb7c", + "metadata": {}, + "outputs": [ + { + "ename": "KeyError", + "evalue": "'ANKI_USER'", + "output_type": "error", + "traceback": [ + "\u001b[31m----------------------------------------------------------\u001b[39m", + "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[22]\u001b[39m\u001b[32m, line 2\u001b[39m", + "\u001b[32m 1\u001b[39m \u001b[38;5;66;03m#| eval: false\u001b[39;00m", + "\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m sync(user=os.environ[\u001b[33m'ANKI_USER'\u001b[39m], passw=os.environ[\u001b[33m'ANKI_PASS'\u001b[39m]) \u001b[38;5;66;03m# first time\u001b[39;00m", + "\u001b[32m 3\u001b[39m sync() \u001b[38;5;66;03m# after that\u001b[39;00m", + "", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.12-macos-aarch64-none/lib/python3.12/os.py:714\u001b[39m, in \u001b[36m_Environ.__getitem__\u001b[39m\u001b[34m(self, key)\u001b[39m", + "\u001b[32m 711\u001b[39m value = \u001b[38;5;28mself\u001b[39m._data[\u001b[38;5;28mself\u001b[39m.encodekey(key)]", + "\u001b[32m 712\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:", + "\u001b[32m 713\u001b[39m \u001b[38;5;66;03m# raise KeyError with the original key value\u001b[39;00m", + "\u001b[32m--> \u001b[39m\u001b[32m714\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m", + "\u001b[32m 715\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m.decodevalue(value)", + "", + "\u001b[31mKeyError\u001b[39m: 'ANKI_USER'" + ] + } + ], "source": [ "#| eval: false\n", "sync(user=os.environ['ANKI_USER'], passw=os.environ['ANKI_PASS']) # first time\n", @@ -418,7 +535,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "&`[add_fb_card, find_notes, find_note_ids, find_cards, find_card_ids, get_note, del_card, update_fb_note, sync]`\n" + "&`[add_card, add_fb_card, add_cloze_card, find_notes, find_note_ids, find_cards, find_card_ids, get_note, del_note, update_fb_note, next_card, answer_buttons, answer_card, due_counts, sync]`\n" ] } ], @@ -536,7 +653,18 @@ "execution_count": null, "id": "b437fd66", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'collection.anki2'" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "col = Collection.open()\n", "col.path.name" @@ -547,7 +675,18 @@ "execution_count": null, "id": "50425ffa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(['Basic', 'Cloze'], ['Default'])" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "col.notetypes(), col.decks()" ] @@ -557,7 +696,25 @@ "execution_count": null, "id": "908f68b6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "
\n", + "\n", + "**Front**: adiós | **Back**: goodbye | 🏷 spanish\n", + "\n", + "
" + ], + "text/plain": [ + "Note(1784904710932, Front='adiós', Back='goodbye', tags=['spanish'])" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "n = col.add(Front='adiós', Back='goodbye', deck='Spanish::Vocab', tags=['spanish'])\n", "n" @@ -568,7 +725,18 @@ "execution_count": null, "id": "b9783973", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(1, 0, 0)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "col.due_counts('Spanish')" ] @@ -578,7 +746,18 @@ "execution_count": null, "id": "2329e024", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[Note(1784904710932, Front='adiós', Back='goodbye', tags=['spanish'])]" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "col.find_notes(deck='Spanish')" ] diff --git a/tests/test_sync.py b/tests/test_sync.py index 2d2707c..f199a88 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -98,6 +98,47 @@ def test_round_trip(server, tmp_path): col.close() assert ok +def test_review_sync(server, tmp_path): + "Reviews made in fastanki reach other clients through the sync server: card state, revlog, and daily counters" + ep = server + a = Collection.open(tmp_path/'a'/'collection.anki2') + n = a.add(Front='study', Back='me') + cid = a.find_card_ids()[0] + a.answer_card(cid, 3, taken_ms=1234) + a.answer_card(cid, 3) # graduates to a 1-day review card + assert a.sync(user='tester', passw='s3kret', endpoint=ep, upload=True) == 'full sync' + a.answer_card(cid, 1) # a lapse afterwards, delta-synced + assert a.sync() == 'success' + + b = Collection.open(tmp_path/'b'/'collection.anki2') + assert b.sync(user='tester', passw='s3kret', endpoint=ep) == 'full sync' + assert [tuple(r) for r in b.q('select ease, type from revlog where cid=? order by id', cid)] == [(3,0),(3,0),(1,1)] + c = b.find_cards(Front='study')[0] + assert (c.type, c.queue, c.lapses) == (3, 1, 1) # mid-relearning, intraday + + from fastanki._proto import decks_pb2 + cmn = decks_pb2.Deck.Common() + cmn.ParseFromString(b.q1('select common from decks where id=1')) + assert (cmn.new_studied, cmn.review_studied) == (1, 1) + assert cmn.milliseconds_studied >= 1234 + + # the other direction: b finishes relearning, and a picks it up by delta + b.answer_card(cid, 3) + assert b.sync() == 'success' + assert a.sync() == 'success' + assert a.q1('select count(*) from revlog where cid=?', cid) == 4 + assert a.q1('select type from cards where id=?', cid) == 2 + + # and desktop Anki accepts the reviewed collection wholesale + b.close() + oc = AnkiCollection(str(b.path)) + ok = oc.fix_integrity()[1] + assert oc.db.execute('select count(*) from revlog')[0][0] == 4 + oc.close() + a.close() + assert ok + + # ---- fastanki-on-both-ends: conflict merge and full-sync bookkeeping ---- def _up(col, ep): return col.sync(user='tester', passw='s3kret', endpoint=ep, upload=True) @@ -224,7 +265,8 @@ def test_tool_schemas(): from toolslm.funccall import get_schema tools = [fastanki.add_card, fastanki.add_fb_card, fastanki.add_cloze_card, fastanki.find_notes, fastanki.find_note_ids, fastanki.find_cards, fastanki.find_card_ids, fastanki.get_note, - fastanki.del_note, fastanki.update_fb_note, fastanki.sync] + fastanki.del_note, fastanki.update_fb_note, fastanki.next_card, fastanki.answer_buttons, + fastanki.answer_card, fastanki.due_counts, fastanki.sync] for f in tools: props = get_schema(f)['input_schema']['properties'] # raises if a param can't be schema'd assert props and all(v.get('description') for v in props.values()), f.__name__