From f095f4f1c7930a3611ce0b083578917947f8d48b Mon Sep 17 00:00:00 2001 From: Ryan Barrett Date: Mon, 23 Mar 2026 08:24:30 -0700 Subject: [PATCH] =?UTF-8?q?as2.to=5Fas1:=20replace=20custom=20emoji=20shor?= =?UTF-8?q?tcodes=20with=20Unicode=20replacement=20char=20=EF=BF=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit for snarfed/bridgy-fed#1104 not ready yet! I think this also needs to handle other tags with indices and shuffle them over too, but it doesn't yet --- README.md | 1 + granary/as2.py | 66 ++++++++++++++++++++++++- granary/tests/test_as2.py | 101 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d27a4808..fecd4fcb 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,7 @@ Changelog * `as2` * `to_as1`: * Handle multiply-valued `content`. + * Replace custom emoji shortcodes with Unicode replacement char � ([snarfed/bridgy-fed#1104](https://github.com/snarfed/bridgy-fed/issues/1104)). * `bluesky`: * Convert `application` and `service` actors to/from the Bluesky `bot` self-label. * `from_as1`: diff --git a/granary/as2.py b/granary/as2.py index 665cbac5..b642b3e3 100644 --- a/granary/as2.py +++ b/granary/as2.py @@ -69,6 +69,7 @@ def _invert(d): 'audio': 'Audio', 'collection': 'Collection', 'comment': 'Note', + 'emoji': 'Emoji', # not in either AS1 or AS2 spec 'event': 'Event', 'group': 'Group', 'hashtag': 'Tag', # not in AS2 spec; needed for correct round trip conversion @@ -581,7 +582,7 @@ def collection_to_as1(field): icons = util.pop_list(obj, 'icon') images = util.pop_list(obj, 'image') - if type in ACTOR_TYPES: + if type in ACTOR_TYPES or type == 'Emoji': if images: # by convention, first element in AS2 images field is banner/header if isinstance(images[0], str): @@ -644,6 +645,8 @@ def collection_to_as1(field): attachments = all_to_as1('attachment', plural=True) tags_as1 = [] quote_urls = [] + # Emoji tags: maps string shortcode to AS1 tag. to replace in content/displayName. + emoji_tags = {} for tag in util.pop_list(obj, 'tag'): if isinstance(tag, str): tags_as1.append(tag) @@ -675,6 +678,15 @@ def collection_to_as1(field): obj['content']) continue + elif tag.get('type') == 'Emoji': + # https://github.com/snarfed/bridgy-fed/issues/1104 + # https://docs.joinmastodon.org/spec/activitypub/#emoji + as1_tag = to_as1(tag) + if shortcode := tag.get('name'): + emoji_tags[shortcode] = as1_tag + else: + tags_as1.append(as1_tag) + else: # other tag tags_as1.append(to_as1({ @@ -683,6 +695,58 @@ def collection_to_as1(field): **tag, })) + # replace custom emoji shortcodes (e.g. :myemoji:) with � (Unicode replacement + # character, \uFFFD) in content, contentMap, and displayName. preserve tags, add + # indices. + # https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + if emoji_tags: + # build a combined regex pattern for all shortcodes + pattern = re.compile('|'.join(re.escape(code) for code in emoji_tags.keys())) + + def replace_shortcodes(text): + """Replace emoji shortcodes with REPLACEMENT_CHAR, return (new_text, positions). + + positions maps shortcode -> list of startIndex values in the result string. + """ + if not text: + return text, {} + + parts = [] + last_end = 0 + pos = 0 + positions = {} + + for match in pattern.finditer(text): + before = text[last_end:match.start()] + parts.append(before) + pos += len(before) + sc = match.group(0) + positions.setdefault(sc, []).append(pos) + parts.append('�') + pos += 1 + last_end = match.end() + + parts.append(text[last_end:]) + return ''.join(parts), positions + + content, positions = replace_shortcodes(obj.get('content', '')) + if content != obj.get('content', ''): + obj['content'] = content + for lang, val in obj.get('contentMap', {}).items(): + obj['contentMap'][lang], _ = replace_shortcodes(val) + + if displayName: + new_dn, _ = replace_shortcodes(displayName) + displayName = new_dn.strip() + + # add emoji tags to tags_as1, with startIndex/length where available + for shortcode, tag in emoji_tags.items(): + if starts := positions.get(shortcode): + for start in starts: + tags_as1.append({**tag, 'startIndex': start, 'length': 1}) + else: + tags_as1.append(tag) + # check quote post fields on the top level object # https://misskey-hub.net/ns#_misskey_quote # https://socialhub.activitypub.rocks/t/repost-share-with-quote-a-k-a-attach-someone-elses-post-to-your-own-post/659/19 diff --git a/granary/tests/test_as2.py b/granary/tests/test_as2.py index 93735e98..c05db9c8 100644 --- a/granary/tests/test_as2.py +++ b/granary/tests/test_as2.py @@ -920,6 +920,107 @@ def test_to_as1_replies_list(self): }], })) + def test_to_as1_emoji_in_note_content(self): + # custom emoji shortcodes in Note content should be replaced with + # REPLACEMENT_CHAR, and the emoji tags should include startIndex/length + # https://github.com/snarfed/bridgy-fed/issues/1104 + self.assert_equals({ + 'objectType': 'note', + 'content': 'hello � world � �', + 'contentMap': {'en': 'hello � world � �'}, + 'tags': [{ + 'objectType': 'emoji', + 'displayName': ':wave:', + 'startIndex': 6, + 'length': 1, + 'image': [{'url': 'https://example.com/wave.png'}], + }, { + 'objectType': 'emoji', + 'displayName': ':wave:', + 'startIndex': 14, + 'length': 1, + 'image': [{'url': 'https://example.com/wave.png'}], + }, { + 'objectType': 'emoji', + 'displayName': ':blobcat:', + 'startIndex': 16, + 'length': 1, + 'image': [{'url': 'https://example.com/blobcat.png'}], + }], + }, as2.to_as1({ + 'type': 'Note', + 'content': 'hello :wave: world :wave: :blobcat:', + 'contentMap': {'en': 'hello :wave: world :wave: :blobcat:'}, + 'tag': [{ + 'type': 'Emoji', + 'name': ':wave:', + 'icon': {'type': 'Image', 'url': 'https://example.com/wave.png'}, + }, { + 'type': 'Emoji', + 'name': ':blobcat:', + 'icon': {'type': 'Image', 'url': 'https://example.com/blobcat.png'}, + }], + })) + + def test_to_as1_emoji_in_actor_name(self): + self.assert_equals({ + 'objectType': 'person', + 'displayName': 'Alice � Dev', + 'tags': [{ + 'objectType': 'emoji', + 'displayName': ':sparkles:', + 'image': [{'url': 'https://example.com/sparkles.png'}], + }], + }, as2.to_as1({ + 'type': 'Person', + 'name': 'Alice :sparkles: Dev', + 'tag': [{ + 'type': 'Emoji', + 'name': ':sparkles:', + 'icon': {'type': 'Image', 'url': 'https://example.com/sparkles.png'}, + }], + })) + + def test_to_as1_emoji_at_edges_of_name(self): + # emoji at start/end of actor name: whitespace should be stripped + self.assert_equals({ + 'objectType': 'person', + 'displayName': '� Alice �', + 'tags': [{ + 'objectType': 'emoji', + 'displayName': ':fire:', + 'image': [{'url': 'https://example.com/fire.png'}], + }], + }, as2.to_as1({ + 'type': 'Person', + 'name': ':fire: Alice :fire:', + 'tag': [{ + 'type': 'Emoji', + 'name': ':fire:', + 'icon': {'type': 'Image', 'url': 'https://example.com/fire.png'}, + }], + })) + + def test_to_as1_emoji_no_shortcode_in_content(self): + # emoji tag present but shortcode not in content: tag preserved without indices + self.assert_equals({ + 'objectType': 'note', + 'content': 'hello world', + 'tags': [{ + 'objectType': 'emoji', + 'displayName': ':wave:', + 'image': [{'url': 'https://example.com/wave.png'}], + }], + }, as2.to_as1({ + 'type': 'Note', + 'content': 'hello world', + 'tag': [{ + 'type': 'Emoji', + 'name': ':wave:', + 'icon': {'type': 'Image', 'url': 'https://example.com/wave.png'}, + }], + })) + def test_link_tags_no_indices(self): # no indices, should be a noop obj = {