fix(rtfcre): escape characters outside code page 1252 on export

news.d/bugfix/1705.core.md

@@ -0,0 +1 @@
+Fix RTF dictionary export failing on characters outside code page 1252; they are now escaped with the RTF `\uN` control word.

plover/dictionary/rtfcre_dict.py

@@ -30,6 +30,31 @@
 )
 
 
+def escape_unicode(text):
+    """Escape characters that code page 1252 cannot represent.
+
+    RTF encodes such characters with the `\\uN` control word, using UTF-16
+    code units, so characters outside the BMP are written as a surrogate
+    pair. The escapes are wrapped in a group setting `\\uc0` so readers do
+    not expect an ANSI fallback character.
+    """
+    parts = []
+    for char in text:
+        try:
+            char.encode("cp1252")
+        except UnicodeEncodeError:
+            codepoint = ord(char)
+            if codepoint > 0xFFFF:
+                codepoint -= 0x10000
+                units = (0xD800 + (codepoint >> 10), 0xDC00 + (codepoint & 0x3FF))
+            else:
+                units = (codepoint,)
+            escapes = "".join(rf"\u{unit} " for unit in units)
+            char = rf"{{\uc0{escapes}}}"
+        parts.append(char)
+    return "".join(parts)
+
+
 class RegexFormatter:
     def __init__(self, spec_list, escape_fn):
         self._escape_fn = escape_fn
@@ -115,7 +140,7 @@ def __init__(self):
     def escape(self, text):
         for rx, replacement in self._to_escape:
             text = rx.sub(replacement, text)
-        return text
+        return escape_unicode(text)
 
     def format(self, translation):
         s = self._translation_formatter.format(translation)

plover/dictionary/rtfcre_parse.py

@@ -38,6 +38,8 @@ def finalize_translation(text):
 def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True):
     not_text = r"\{}"
     style_rx = re.compile("s[0-9]+")
+    unicode_rx = re.compile("u-?[0-9]+")
+    uc_rx = re.compile("uc[0-9]+")
     tokenizer = RtfTokenizer(text)
     next_token = tokenizer.next_token
     rewind_token = tokenizer.rewind_token
@@ -46,6 +48,11 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True):
         raise BadRtfError("invalid header")
     # Parse header/document.
     g_destination, g_text = "rtf1", ""
+    # Number of fallback characters written after each `\uN` escape,
+    # per group. 1 is the RTF default.
+    g_uc = 1
+    # Number of fallback characters still to be skipped.
+    skip_count = 0
     group_stack = deque()
     stylesheet = {}
     steno = None
@@ -143,8 +150,9 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True):
                 if stack_depth:
                     break
                 continue
-            group_stack.append((g_destination, g_text))
+            group_stack.append((g_destination, g_text, g_uc))
             g_destination, g_text = destination, ""
+            skip_count = 0
             if rewind:
                 rewind_token(token)
             continue
@@ -206,7 +214,8 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True):
                 stylesheet[g_destination] = g_text
             else:
                 text = g_text
-            g_destination, g_text = group_stack.pop()
+            g_destination, g_text, g_uc = group_stack.pop()
+            skip_count = 0
             g_text += text
             continue
         # Control char/word.
@@ -240,7 +249,29 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True):
                 "cxfl": "{>}",
             }.get(ctrl)
             if text is not None:
-                g_text += text
+                if skip_count:
+                    skip_count -= 1
+                else:
+                    g_text += text
+            # Unicode escape.
+            elif unicode_rx.fullmatch(ctrl):
+                code_unit = int(ctrl[1:]) & 0xFFFF
+                if (
+                    0xDC00 <= code_unit < 0xE000
+                    and g_text
+                    and 0xD800 <= ord(g_text[-1]) < 0xDC00
+                ):
+                    # Low surrogate: combine with the preceding high one.
+                    high = ord(g_text[-1]) - 0xD800
+                    g_text = g_text[:-1] + chr(
+                        0x10000 + (high << 10) + (code_unit - 0xDC00)
+                    )
+                else:
+                    g_text += chr(code_unit)
+                skip_count = g_uc
+            # Number of fallback characters following a unicode escape.
+            elif uc_rx.fullmatch(ctrl):
+                g_uc = int(ctrl[2:])
             # Delete Spaces.
             elif ctrl == "cxds":
                 token = next_token()
@@ -284,6 +315,12 @@ def parse_rtfcre(text, normalize=lambda s: s, skip_errors=True):
             continue
         # Text.
         text = token
+        if skip_count:
+            skipped = min(skip_count, len(text))
+            skip_count -= skipped
+            text = text[skipped:]
+            if not text:
+                continue
         token = next_token()
         if token == r"\cxds":
             # Suffix.

test/test_rtfcre_dict.py

@@ -77,6 +77,10 @@
             r"=macro{\*\cxplovermeta <-ceci n'est pas une macro}",
         ),
         lambda: ("{*}something", r"{\*\cxplovermeta *}something"),
+        # Characters outside code page 1252 are escaped as UTF-16 code units.
+        lambda: ("ph\u1edf", r"ph{\uc0\u7903 }"),
+        lambda: ("\uc18d", r"{\uc0\u49549 }"),
+        lambda: ("\U0001f60a", r"{\uc0\u55357 \u56842 }"),
     )
 )
 def test_format_translation(before, expected):
@@ -456,6 +460,37 @@ def rtf_load_test(*spec, xfail=False):
         '2': '2',
         """
     ),
+    # Unicode escapes, including surrogate pairs.
+    lambda: rtf_load_test(
+        r"""
+        {\*\cxs TPA*}ph{\uc0\u7903 }
+
+        'TPA*': 'ph\u1edf',
+        """
+    ),
+    lambda: rtf_load_test(
+        r"""
+        {\*\cxs SPHAOEUL}{\uc0\u55357 \u56842 }
+
+        'SPHAOEUL': '\U0001f60a',
+        """
+    ),
+    # Negative values, as allowed by the RTF spec.
+    lambda: rtf_load_test(
+        r"""
+        {\*\cxs KPWHA}{\uc0\u-15987 }
+
+        'KPWHA': '\uc18d',
+        """
+    ),
+    # `\ucN` fallback characters after an escape are skipped.
+    lambda: rtf_load_test(
+        r"""
+        {\*\cxs TPA*}ph\u7903 ?
+
+        'TPA*': 'ph\u1edf',
+        """
+    ),
 )
 
 
@@ -513,6 +548,17 @@ def rtf_save_test(dict_entries, rtf_entries):
         """,
         (rb"{\*\cxs PHROLG}{\*\cxplovermeta PLOVER:TOGGLE}",),
     ),
+    # Characters outside code page 1252 must not break saving.
+    lambda: rtf_save_test(
+        """
+        "TPA*": "ph\u1edf",
+        "SPHAOEUL": "\U0001f60a",
+        """,
+        (
+            rb"{\*\cxs TPA*}ph{\uc0\u7903 }",
+            rb"{\*\cxs SPHAOEUL}{\uc0\u55357 \u56842 }",
+        ),
+    ),
 )