fix(rtfcre): escape characters outside code page 1252 on export

news.d/bugfix/1705.core.md

@@ -0,0 +1 @@
+Fix RTF dictionary export failing on characters outside code page 1252; they are now escaped with the RTF `\uN` control word.

plover/dictionary/rtfcre_dict.py

@@ -30,6 +30,31 @@
 )
 
 
+def escape_unicode(text):
+    """Escape characters that code page 1252 cannot represent.
+
+    RTF encodes such characters with the `\\uN` control word, using UTF-16
+    code units, so characters outside the BMP are written as a surrogate
+    pair. The escapes are wrapped in a group setting `\\uc0` so readers do
+    not expect an ANSI fallback character.
+    """
+    parts = []
+    for char in text:
+        try:
+            char.encode("cp1252")
+        except UnicodeEncodeError:
+            codepoint = ord(char)
+            if codepoint > 0xFFFF:
+                codepoint -= 0x10000
+                units = (0xD800 + (codepoint >> 10), 0xDC00 + (codepoint & 0x3FF))
+            else:
+                units = (codepoint,)
+            escapes = "".join(rf"\u{unit} " for unit in units)
+            char = rf"{{\uc0{escapes}}}"
+        parts.append(char)
+    return "".join(parts)
+
+
 class RegexFormatter:
     def __init__(self, spec_list, escape_fn):
         self._escape_fn = escape_fn
@@ -115,7 +140,7 @@ def __init__(self):
     def escape(self, text):
         for rx, replacement in self._to_escape:
             text = rx.sub(replacement, text)
-        return text
+        return escape_unicode(text)
 
     def format(self, translation):
         s = self._translation_formatter.format(translation)

plover/dictionary/rtfcre_parse.py

@@ -38,6 +38,8 @@ def finalize_translation(text):
 def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True):
     not_text = r"\{}"
     style_rx = re.compile("s[0-9]+")
+    unicode_rx = re.compile("u-?[0-9]+")
+    uc_rx = re.compile("uc[0-9]+")
     tokenizer = RtfTokenizer(text)
     next_token = tokenizer.next_token
     rewind_token = tokenizer.rewind_token
@@ -241,6 +243,24 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True):
             }.get(ctrl)
             if text is not None:
                 g_text += text
+            # Unicode escape.
+            elif unicode_rx.fullmatch(ctrl):
+                code_unit = int(ctrl[1:]) & 0xFFFF
+                if (
+                    0xDC00 <= code_unit < 0xE000
+                    and g_text
+                    and 0xD800 <= ord(g_text[-1]) < 0xDC00
+                ):
+                    # Low surrogate: combine with the preceding high one.
+                    high = ord(g_text[-1]) - 0xD800
+                    g_text = g_text[:-1] + chr(
+                        0x10000 + (high << 10) + (code_unit - 0xDC00)
+                    )
+                else:
+                    g_text += chr(code_unit)
+            # Number of fallback characters following a unicode escape.
+            elif uc_rx.fullmatch(ctrl):
+                pass
             # Delete Spaces.
             elif ctrl == "cxds":
                 token = next_token()

test/test_rtfcre_dict.py

@@ -77,6 +77,10 @@
             r"=macro{\*\cxplovermeta <-ceci n'est pas une macro}",
         ),
         lambda: ("{*}something", r"{\*\cxplovermeta *}something"),
+        # Characters outside code page 1252 are escaped as UTF-16 code units.
+        lambda: ("ph\u1edf", r"ph{\uc0\u7903 }"),
+        lambda: ("\uc18d", r"{\uc0\u49549 }"),
+        lambda: ("\U0001f60a", r"{\uc0\u55357 \u56842 }"),
     )
 )
 def test_format_translation(before, expected):
@@ -456,6 +460,29 @@ def rtf_load_test(*spec, xfail=False):
         '2': '2',
         """
     ),
+    # Unicode escapes, including surrogate pairs.
+    lambda: rtf_load_test(
+        r"""
+        {\*\cxs TPA*}ph{\uc0\u7903 }
+
+        'TPA*': 'ph\u1edf',
+        """
+    ),
+    lambda: rtf_load_test(
+        r"""
+        {\*\cxs SPHAOEUL}{\uc0\u55357 \u56842 }
+
+        'SPHAOEUL': '\U0001f60a',
+        """
+    ),
+    # Negative values, as allowed by the RTF spec.
+    lambda: rtf_load_test(
+        r"""
+        {\*\cxs KPWHA}{\uc0\u-15987 }
+
+        'KPWHA': '\uc18d',
+        """
+    ),
 )
 
 
@@ -513,6 +540,17 @@ def rtf_save_test(dict_entries, rtf_entries):
         """,
         (rb"{\*\cxs PHROLG}{\*\cxplovermeta PLOVER:TOGGLE}",),
     ),
+    # Characters outside code page 1252 must not break saving.
+    lambda: rtf_save_test(
+        """
+        "TPA*": "ph\u1edf",
+        "SPHAOEUL": "\U0001f60a",
+        """,
+        (
+            rb"{\*\cxs TPA*}ph{\uc0\u7903 }",
+            rb"{\*\cxs SPHAOEUL}{\uc0\u55357 \u56842 }",
+        ),
+    ),
 )