Fix within surrogate detection

telethon/extensions/html.py

@@ -174,12 +174,10 @@ def unparse(text: str, entities: Iterable[TypeMessageEntity], _offset: int = 0,
         # Otherwise we would end up with malformed text and fail to encode.
         # For example of bad input: "Hi \ud83d\ude1c"
         # https://en.wikipedia.org/wiki/UTF-16#U+010000_to_U+10FFFF
-        while (relative_offset < _length
-                and '\ud800' <= text[relative_offset] <= '\udfff'):
+        while helpers.within_surrogate(text, relative_offset, length=_length):
             relative_offset += 1
 
-        while (relative_offset + length < _length
-                and '\ud800' <= text[relative_offset + length] <= '\udfff'):
+        while helpers.within_surrogate(text, relative_offset + length, length=_length):
             length += 1
 
         entity_text = unparse(text=text[relative_offset:relative_offset + length],
@@ -224,7 +222,7 @@ def unparse(text: str, entities: Iterable[TypeMessageEntity], _offset: int = 0,
             skip_entity = True
         last_offset = relative_offset + (0 if skip_entity else length)
 
-    while last_offset < _length and '\ud800' <= text[last_offset] <= '\udfff':
+    while helpers.within_surrogate(text, last_offset, length=_length):
         last_offset += 1
 
     html.append(escape(text[last_offset:]))

telethon/extensions/markdown.py

@@ -6,7 +6,7 @@ since they seem to count as two characters and it's a bit strange.
 import re
 import warnings
 
-from ..helpers import add_surrogate, del_surrogate, strip_text
+from ..helpers import add_surrogate, del_surrogate, within_surrogate, strip_text
 from ..tl import TLObject
 from ..tl.types import (
     MessageEntityBold, MessageEntityItalic, MessageEntityCode,
@@ -185,11 +185,11 @@ def unparse(text, entities, delimiters=None, url_fmt=None):
     while insert_at:
         at, what = insert_at.pop()
 
-        # If we are in the middle of a surrogate nudge the position by +1.
+        # If we are in the middle of a surrogate nudge the position by -1.
         # Otherwise we would end up with malformed text and fail to encode.
         # For example of bad input: "Hi \ud83d\ude1c"
         # https://en.wikipedia.org/wiki/UTF-16#U+010000_to_U+10FFFF
-        while at < len(text) and '\ud800' <= text[at] <= '\udfff':
+        while within_surrogate(text, at):
             at += 1
 
         text = text[:at] + what + text[at:]

telethon/helpers.py

@@ -40,6 +40,20 @@ def del_surrogate(text):
     return text.encode('utf-16', 'surrogatepass').decode('utf-16')
 
 
+def within_surrogate(text, index, *, length=None):
+    """
+    `True` if ``index`` is within a surrogate (before and after it, not at!).
+    """
+    if length is None:
+        length = len(text)
+
+    return (
+            1 < index < len(text) and  # in bounds
+            '\ud800' <= text[index - 1] <= '\udfff' and  # previous is
+            '\ud800' <= text[index] <= '\udfff'  # current is
+    )
+
+
 def strip_text(text, entities):
     """
     Strips whitespace from the given text modifying the provided entities.