From 2c0ce279932e1a31700d391e8fd1454f4134facc Mon Sep 17 00:00:00 2001 From: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:27:44 -0700 Subject: [PATCH 1/2] fix(rtfcre): escape characters outside code page 1252 on export Saving a dictionary as RTF encodes the file as code page 1252, so any translation containing a character outside that code page raised UnicodeEncodeError and the export failed. The RTF spec encodes such characters with the \uN control word, using UTF-16 code units, so characters outside the BMP become a surrogate pair. Escape them on save, wrapped in a group setting \uc0, and handle \uN (and \ucN) when parsing so RTF dictionaries still round-trip. Tests cover formatting, saving and loading of BMP characters, surrogate pairs, and the negative \uN values the spec allows. --- news.d/bugfix/1705.core.md | 1 + plover/dictionary/rtfcre_dict.py | 27 +++++++++++++++++++++- plover/dictionary/rtfcre_parse.py | 20 ++++++++++++++++ test/test_rtfcre_dict.py | 38 +++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 news.d/bugfix/1705.core.md diff --git a/news.d/bugfix/1705.core.md b/news.d/bugfix/1705.core.md new file mode 100644 index 000000000..9fc5e1696 --- /dev/null +++ b/news.d/bugfix/1705.core.md @@ -0,0 +1 @@ +Fix RTF dictionary export failing on characters outside code page 1252; they are now escaped with the RTF `\uN` control word. diff --git a/plover/dictionary/rtfcre_dict.py b/plover/dictionary/rtfcre_dict.py index 797aefb96..77f361e46 100644 --- a/plover/dictionary/rtfcre_dict.py +++ b/plover/dictionary/rtfcre_dict.py @@ -30,6 +30,31 @@ ) +def escape_unicode(text): + """Escape characters that code page 1252 cannot represent. + + RTF encodes such characters with the `\\uN` control word, using UTF-16 + code units, so characters outside the BMP are written as a surrogate + pair. The escapes are wrapped in a group setting `\\uc0` so readers do + not expect an ANSI fallback character. + """ + parts = [] + for char in text: + try: + char.encode("cp1252") + except UnicodeEncodeError: + codepoint = ord(char) + if codepoint > 0xFFFF: + codepoint -= 0x10000 + units = (0xD800 + (codepoint >> 10), 0xDC00 + (codepoint & 0x3FF)) + else: + units = (codepoint,) + escapes = "".join(rf"\u{unit} " for unit in units) + char = rf"{{\uc0{escapes}}}" + parts.append(char) + return "".join(parts) + + class RegexFormatter: def __init__(self, spec_list, escape_fn): self._escape_fn = escape_fn @@ -115,7 +140,7 @@ def __init__(self): def escape(self, text): for rx, replacement in self._to_escape: text = rx.sub(replacement, text) - return text + return escape_unicode(text) def format(self, translation): s = self._translation_formatter.format(translation) diff --git a/plover/dictionary/rtfcre_parse.py b/plover/dictionary/rtfcre_parse.py index 520052522..ae7c06678 100644 --- a/plover/dictionary/rtfcre_parse.py +++ b/plover/dictionary/rtfcre_parse.py @@ -38,6 +38,8 @@ def finalize_translation(text): def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True): not_text = r"\{}" style_rx = re.compile("s[0-9]+") + unicode_rx = re.compile("u-?[0-9]+") + uc_rx = re.compile("uc[0-9]+") tokenizer = RtfTokenizer(text) next_token = tokenizer.next_token rewind_token = tokenizer.rewind_token @@ -241,6 +243,24 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True): }.get(ctrl) if text is not None: g_text += text + # Unicode escape. + elif unicode_rx.fullmatch(ctrl): + code_unit = int(ctrl[1:]) & 0xFFFF + if ( + 0xDC00 <= code_unit < 0xE000 + and g_text + and 0xD800 <= ord(g_text[-1]) < 0xDC00 + ): + # Low surrogate: combine with the preceding high one. + high = ord(g_text[-1]) - 0xD800 + g_text = g_text[:-1] + chr( + 0x10000 + (high << 10) + (code_unit - 0xDC00) + ) + else: + g_text += chr(code_unit) + # Number of fallback characters following a unicode escape. + elif uc_rx.fullmatch(ctrl): + pass # Delete Spaces. elif ctrl == "cxds": token = next_token() diff --git a/test/test_rtfcre_dict.py b/test/test_rtfcre_dict.py index 51249b836..5bd5bf15b 100644 --- a/test/test_rtfcre_dict.py +++ b/test/test_rtfcre_dict.py @@ -77,6 +77,10 @@ r"=macro{\*\cxplovermeta <-ceci n'est pas une macro}", ), lambda: ("{*}something", r"{\*\cxplovermeta *}something"), + # Characters outside code page 1252 are escaped as UTF-16 code units. + lambda: ("ph\u1edf", r"ph{\uc0\u7903 }"), + lambda: ("\uc18d", r"{\uc0\u49549 }"), + lambda: ("\U0001f60a", r"{\uc0\u55357 \u56842 }"), ) ) def test_format_translation(before, expected): @@ -456,6 +460,29 @@ def rtf_load_test(*spec, xfail=False): '2': '2', """ ), + # Unicode escapes, including surrogate pairs. + lambda: rtf_load_test( + r""" + {\*\cxs TPA*}ph{\uc0\u7903 } + + 'TPA*': 'ph\u1edf', + """ + ), + lambda: rtf_load_test( + r""" + {\*\cxs SPHAOEUL}{\uc0\u55357 \u56842 } + + 'SPHAOEUL': '\U0001f60a', + """ + ), + # Negative values, as allowed by the RTF spec. + lambda: rtf_load_test( + r""" + {\*\cxs KPWHA}{\uc0\u-15987 } + + 'KPWHA': '\uc18d', + """ + ), ) @@ -513,6 +540,17 @@ def rtf_save_test(dict_entries, rtf_entries): """, (rb"{\*\cxs PHROLG}{\*\cxplovermeta PLOVER:TOGGLE}",), ), + # Characters outside code page 1252 must not break saving. + lambda: rtf_save_test( + """ + "TPA*": "ph\u1edf", + "SPHAOEUL": "\U0001f60a", + """, + ( + rb"{\*\cxs TPA*}ph{\uc0\u7903 }", + rb"{\*\cxs SPHAOEUL}{\uc0\u55357 \u56842 }", + ), + ), ) From 33f5780ec81b266c49933567e11e7818d82d48b1 Mon Sep 17 00:00:00 2001 From: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:27:56 -0700 Subject: [PATCH 2/2] fix(rtfcre): skip \ucN fallback characters after unicode escapes \ucN sets how many fallback characters follow each \uN escape, and 1 is the RTF default, so files written by other programs emit `\u7903 ?` and the `?` was ending up in the translation. Track the count per group, restore it on group end, and drop that many following characters (control-word substitutions and plain text alike). --- plover/dictionary/rtfcre_parse.py | 25 +++++++++++++++++++++---- test/test_rtfcre_dict.py | 8 ++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/plover/dictionary/rtfcre_parse.py b/plover/dictionary/rtfcre_parse.py index ae7c06678..7b7c3d430 100644 --- a/plover/dictionary/rtfcre_parse.py +++ b/plover/dictionary/rtfcre_parse.py @@ -48,6 +48,11 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True): raise BadRtfError("invalid header") # Parse header/document. g_destination, g_text = "rtf1", "" + # Number of fallback characters written after each `\uN` escape, + # per group. 1 is the RTF default. + g_uc = 1 + # Number of fallback characters still to be skipped. + skip_count = 0 group_stack = deque() stylesheet = {} steno = None @@ -145,8 +150,9 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True): if stack_depth: break continue - group_stack.append((g_destination, g_text)) + group_stack.append((g_destination, g_text, g_uc)) g_destination, g_text = destination, "" + skip_count = 0 if rewind: rewind_token(token) continue @@ -208,7 +214,8 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True): stylesheet[g_destination] = g_text else: text = g_text - g_destination, g_text = group_stack.pop() + g_destination, g_text, g_uc = group_stack.pop() + skip_count = 0 g_text += text continue # Control char/word. @@ -242,7 +249,10 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True): "cxfl": "{>}", }.get(ctrl) if text is not None: - g_text += text + if skip_count: + skip_count -= 1 + else: + g_text += text # Unicode escape. elif unicode_rx.fullmatch(ctrl): code_unit = int(ctrl[1:]) & 0xFFFF @@ -258,9 +268,10 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True): ) else: g_text += chr(code_unit) + skip_count = g_uc # Number of fallback characters following a unicode escape. elif uc_rx.fullmatch(ctrl): - pass + g_uc = int(ctrl[2:]) # Delete Spaces. elif ctrl == "cxds": token = next_token() @@ -304,6 +315,12 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True): continue # Text. text = token + if skip_count: + skipped = min(skip_count, len(text)) + skip_count -= skipped + text = text[skipped:] + if not text: + continue token = next_token() if token == r"\cxds": # Suffix. diff --git a/test/test_rtfcre_dict.py b/test/test_rtfcre_dict.py index 5bd5bf15b..cf5bf5b8d 100644 --- a/test/test_rtfcre_dict.py +++ b/test/test_rtfcre_dict.py @@ -483,6 +483,14 @@ def rtf_load_test(*spec, xfail=False): 'KPWHA': '\uc18d', """ ), + # `\ucN` fallback characters after an escape are skipped. + lambda: rtf_load_test( + r""" + {\*\cxs TPA*}ph\u7903 ? + + 'TPA*': 'ph\u1edf', + """ + ), )