as2.to_as1: replace custom emoji shortcodes with Unicode replacement char �

README.md

@@ -315,6 +315,7 @@ Changelog
 * `as2`
   * `to_as1`:
     * Handle multiply-valued `content`.
+    * Replace custom emoji shortcodes with Unicode replacement char � ([snarfed/bridgy-fed#1104](https://github.com/snarfed/bridgy-fed/issues/1104)).
 * `bluesky`:
   * Convert `application` and `service` actors to/from the Bluesky `bot` self-label.
   * `from_as1`:

granary/as2.py

@@ -69,6 +69,7 @@ def _invert(d):
   'audio': 'Audio',
   'collection': 'Collection',
   'comment': 'Note',
+  'emoji': 'Emoji',  # not in either AS1 or AS2 spec
   'event': 'Event',
   'group': 'Group',
   'hashtag': 'Tag',  # not in AS2 spec; needed for correct round trip conversion
@@ -581,7 +582,7 @@ def collection_to_as1(field):
   icons = util.pop_list(obj, 'icon')
   images = util.pop_list(obj, 'image')
 
-  if type in ACTOR_TYPES:
+  if type in ACTOR_TYPES or type == 'Emoji':
     if images:
       # by convention, first element in AS2 images field is banner/header
       if isinstance(images[0], str):
@@ -644,6 +645,8 @@ def collection_to_as1(field):
   attachments = all_to_as1('attachment', plural=True)
   tags_as1 = []
   quote_urls = []
+  # Emoji tags: maps string shortcode to AS1 tag. to replace in content/displayName.
+  emoji_tags = {}
   for tag in util.pop_list(obj, 'tag'):
     if isinstance(tag, str):
       tags_as1.append(tag)
@@ -675,6 +678,15 @@ def collection_to_as1(field):
         obj['content'])
       continue
 
+    elif tag.get('type') == 'Emoji':
+      # https://github.com/snarfed/bridgy-fed/issues/1104
+      # https://docs.joinmastodon.org/spec/activitypub/#emoji
+      as1_tag = to_as1(tag)
+      if shortcode := tag.get('name'):
+        emoji_tags[shortcode] = as1_tag
+      else:
+        tags_as1.append(as1_tag)
+
     else:
       # other tag
       tags_as1.append(to_as1({
@@ -683,6 +695,58 @@ def collection_to_as1(field):
         **tag,
       }))
 
+  # replace custom emoji shortcodes (e.g. :myemoji:) with � (Unicode replacement
+  # character, \uFFFD) in content, contentMap, and displayName. preserve tags, add
+  # indices.
+  # https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character
+  if emoji_tags:
+    # build a combined regex pattern for all shortcodes
+    pattern = re.compile('|'.join(re.escape(code) for code in emoji_tags.keys()))
+
+    def replace_shortcodes(text):
+      """Replace emoji shortcodes with REPLACEMENT_CHAR, return (new_text, positions).
+
+      positions maps shortcode -> list of startIndex values in the result string.
+      """
+      if not text:
+        return text, {}
+
+      parts = []
+      last_end = 0
+      pos = 0
+      positions = {}
+
+      for match in pattern.finditer(text):
+        before = text[last_end:match.start()]
+        parts.append(before)
+        pos += len(before)
+        sc = match.group(0)
+        positions.setdefault(sc, []).append(pos)
+        parts.append('�')
+        pos += 1
+        last_end = match.end()
+
+      parts.append(text[last_end:])
+      return ''.join(parts), positions
+
+    content, positions = replace_shortcodes(obj.get('content', ''))
+    if content != obj.get('content', ''):
+      obj['content'] = content
+    for lang, val in obj.get('contentMap', {}).items():
+      obj['contentMap'][lang], _ = replace_shortcodes(val)
+
+    if displayName:
+      new_dn, _ = replace_shortcodes(displayName)
+      displayName = new_dn.strip()
+
+    # add emoji tags to tags_as1, with startIndex/length where available
+    for shortcode, tag in emoji_tags.items():
+      if starts := positions.get(shortcode):
+        for start in starts:
+          tags_as1.append({**tag, 'startIndex': start, 'length': 1})
+      else:
+        tags_as1.append(tag)
+
   # check quote post fields on the top level object
   # https://misskey-hub.net/ns#_misskey_quote
   # https://socialhub.activitypub.rocks/t/repost-share-with-quote-a-k-a-attach-someone-elses-post-to-your-own-post/659/19

granary/tests/test_as2.py

@@ -920,6 +920,107 @@ def test_to_as1_replies_list(self):
       }],
     }))
 
+  def test_to_as1_emoji_in_note_content(self):
+    # custom emoji shortcodes in Note content should be replaced with
+    # REPLACEMENT_CHAR, and the emoji tags should include startIndex/length
+    # https://github.com/snarfed/bridgy-fed/issues/1104
+    self.assert_equals({
+      'objectType': 'note',
+      'content': 'hello � world � �',
+      'contentMap': {'en': 'hello � world � �'},
+      'tags': [{
+        'objectType': 'emoji',
+        'displayName': ':wave:',
+        'startIndex': 6,
+        'length': 1,
+        'image': [{'url': 'https://example.com/wave.png'}],
+      }, {
+        'objectType': 'emoji',
+        'displayName': ':wave:',
+        'startIndex': 14,
+        'length': 1,
+        'image': [{'url': 'https://example.com/wave.png'}],
+      }, {
+        'objectType': 'emoji',
+        'displayName': ':blobcat:',
+        'startIndex': 16,
+        'length': 1,
+        'image': [{'url': 'https://example.com/blobcat.png'}],
+      }],
+    }, as2.to_as1({
+      'type': 'Note',
+      'content': 'hello :wave: world :wave: :blobcat:',
+      'contentMap': {'en': 'hello :wave: world :wave: :blobcat:'},
+      'tag': [{
+        'type': 'Emoji',
+        'name': ':wave:',
+        'icon': {'type': 'Image', 'url': 'https://example.com/wave.png'},
+      }, {
+        'type': 'Emoji',
+        'name': ':blobcat:',
+        'icon': {'type': 'Image', 'url': 'https://example.com/blobcat.png'},
+      }],
+    }))
+
+  def test_to_as1_emoji_in_actor_name(self):
+    self.assert_equals({
+      'objectType': 'person',
+      'displayName': 'Alice � Dev',
+      'tags': [{
+        'objectType': 'emoji',
+        'displayName': ':sparkles:',
+        'image': [{'url': 'https://example.com/sparkles.png'}],
+      }],
+    }, as2.to_as1({
+      'type': 'Person',
+      'name': 'Alice :sparkles: Dev',
+      'tag': [{
+        'type': 'Emoji',
+        'name': ':sparkles:',
+        'icon': {'type': 'Image', 'url': 'https://example.com/sparkles.png'},
+      }],
+    }))
+
+  def test_to_as1_emoji_at_edges_of_name(self):
+    # emoji at start/end of actor name: whitespace should be stripped
+    self.assert_equals({
+      'objectType': 'person',
+      'displayName': '� Alice �',
+      'tags': [{
+        'objectType': 'emoji',
+        'displayName': ':fire:',
+        'image': [{'url': 'https://example.com/fire.png'}],
+      }],
+    }, as2.to_as1({
+      'type': 'Person',
+      'name': ':fire: Alice :fire:',
+      'tag': [{
+        'type': 'Emoji',
+        'name': ':fire:',
+        'icon': {'type': 'Image', 'url': 'https://example.com/fire.png'},
+      }],
+    }))
+
+  def test_to_as1_emoji_no_shortcode_in_content(self):
+    # emoji tag present but shortcode not in content: tag preserved without indices
+    self.assert_equals({
+      'objectType': 'note',
+      'content': 'hello world',
+      'tags': [{
+        'objectType': 'emoji',
+        'displayName': ':wave:',
+        'image': [{'url': 'https://example.com/wave.png'}],
+      }],
+    }, as2.to_as1({
+      'type': 'Note',
+      'content': 'hello world',
+      'tag': [{
+        'type': 'Emoji',
+        'name': ':wave:',
+        'icon': {'type': 'Image', 'url': 'https://example.com/wave.png'},
+      }],
+    }))
+
   def test_link_tags_no_indices(self):
     # no indices, should be a noop
     obj = {