Pass corrected Unicode to XMPP

Emulsion.Telegram/Funogram.fs

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2024 Emulsion contributors <https://github.com/codingteam/emulsion>
+// SPDX-FileCopyrightText: 2025 Emulsion contributors <https://github.com/codingteam/emulsion>
 //
 // SPDX-License-Identifier: MIT
 
@@ -105,16 +105,30 @@ module MessageConverter =
                 pos <- linkEndOffset
             result.Append(text.Substring(pos, text.Length - pos)).ToString()
 
-    let private applyLimits limits text =
+    let private applyLimits limits (text: string) =
         let applyMessageLengthLimit (original: {| text: string; wasLimited: bool |}) =
             match limits.messageLengthLimit with
             | None -> original
             | Some limit when original.text.Length <= limit -> original
             | Some limit ->
-                let newText = original.text.Substring(0,
-                                                      Math.Clamp(limit - limits.dataRedactedMessage.Length,
-                                                                 0,
-                                                                 original.text.Length))
+                assert (limit >= limits.dataRedactedMessage.Length)
+
+                let mutable newTextLength = Math.Clamp(
+                    limit - limits.dataRedactedMessage.Length,
+                    0,
+                    original.text.Length
+                )
+
+                // We should never split surrogate pairs present in the initial message. So, if the message ends with a
+                // high part of such a pair, cut it more, to remove the part of the pair.
+                //
+                // Technically, this will also strip a part of an invalid Unicode sequence if the message originally
+                // contained such an orphan part of the pair without even following it by a high surrogate. But we don't
+                // care.
+                if newTextLength > 0 && Char.IsHighSurrogate(text[newTextLength - 1]) then
+                    newTextLength <- newTextLength - 1
+
+                let newText = original.text.Substring(0, newTextLength)
                 {| text = newText; wasLimited = true |}
 
         let applyLineLimit (original: {| text: string; wasLimited: bool |}) =

Emulsion.Tests/Telegram/FunogramTests.fs

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2024 Emulsion contributors <https://github.com/codingteam/emulsion>
+// SPDX-FileCopyrightText: 2025 Emulsion contributors <https://github.com/codingteam/emulsion>
 //
 // SPDX-License-Identifier: MIT
 
@@ -736,6 +736,21 @@ module FlattenMessageTests =
             flattenMessage replyMessage
         )
 
+    [<Fact>]
+    let ``Flattening should not split surrogate pairs``() =
+        let originalMessage = authoredTelegramMessage "@originalUser" "🐙🐙🐙🐙"
+        let limit = 6
+        let replyMessage = authoredTelegramReplyMessage "@replyingUser" "Reply text" originalMessage.main
+        let flattener = MessageConverter.flatten {
+            MessageConverter.DefaultQuoteSettings with
+                limits.messageLengthLimit = Some limit
+        }
+        let flattened = flattener replyMessage
+        Assert.Equal(
+            Authored { author = "@replyingUser"; text = ">> <@originalUser> 🐙[…]\n\nReply text" },
+            flattened
+        )
+
     [<Fact>]
     let flattenReplyEventMessage() =
         let originalMessage = eventTelegramMessage "@originalUser has entered the chat"

Emulsion.Tests/Xmpp/SharpXmppHelperTests.fs

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2024 Emulsion contributors <https://github.com/codingteam/emulsion>
+// SPDX-FileCopyrightText: 2025 Emulsion contributors <https://github.com/codingteam/emulsion>
 //
 // SPDX-License-Identifier: MIT
 
@@ -16,6 +16,22 @@ open Emulsion.Xmpp
 open Emulsion.Xmpp.SharpXmppHelper.Attributes
 open Emulsion.Xmpp.SharpXmppHelper.Elements
 
+[<Fact>]
+let ``SanitizeXmlText processes emoji as-is``(): unit =
+    Assert.Equal("🐙", SharpXmppHelper.SanitizeXmlText "🐙")
+    Assert.Equal("test🐙", SharpXmppHelper.SanitizeXmlText "test🐙")
+
+[<Fact>]
+let ``SanitizeXmlText replaces parts of UTF-16 surrogate pair with the replacement char``(): unit =
+    let octopus = "🐙"
+    Assert.Equal(2, octopus.Length)
+    let firstHalf = string(octopus[0])
+    let secondHalf = string(octopus[1])
+    Assert.Equal("🐙", firstHalf + secondHalf)
+    Assert.Equal("�", SharpXmppHelper.SanitizeXmlText firstHalf)
+    Assert.Equal("�", SharpXmppHelper.SanitizeXmlText secondHalf)
+    Assert.Equal("test�", SharpXmppHelper.SanitizeXmlText $"test{secondHalf}")
+
 [<Fact>]
 let ``Message body has a proper namespace``() =
     let message = SharpXmppHelper.message "" "cthulhu@test" "text"

Emulsion/Xmpp/SharpXmppHelper.fs

@@ -1,13 +1,16 @@
-// SPDX-FileCopyrightText: 2024 Emulsion contributors <https://github.com/codingteam/emulsion>
+// SPDX-FileCopyrightText: 2025 Emulsion contributors <https://github.com/codingteam/emulsion>
 //
 // SPDX-License-Identifier: MIT
 
 /// Helper functions to deal with SharpXMPP low-level details (such as XML stuff).
 module Emulsion.Xmpp.SharpXmppHelper
 
 open System
+open System.Buffers
+open System.Text
 open System.Xml.Linq
 
+open Microsoft.FSharp.NativeInterop
 open SharpXMPP
 open SharpXMPP.XMPP
 open SharpXMPP.XMPP.Client.Elements
@@ -49,6 +52,28 @@ let private bookmark (roomJid: string) (nickname: string) (password: string opti
     room.Add(nickElement)
     room
 
+#nowarn "9" // for NativePtr
+let SanitizeXmlText(text: string): string =
+    let mutable hasError = false
+    let mutable span = text.AsSpan()
+    while not hasError && not span.IsEmpty do
+        let mutable rune = Rune()
+        let mutable consumed = 0
+        if Rune.DecodeFromUtf16(span, &rune, &consumed) = OperationStatus.Done
+        then span <- span.Slice consumed
+        else hasError <- true
+
+    if hasError then
+        let builder = StringBuilder()
+        for r in text.EnumerateRunes() do
+            let length = r.Utf16SequenceLength
+            let buf = Span(NativePtr.stackalloc<char> length |> NativePtr.toVoidPtr, length)
+            r.EncodeToUtf16 buf |> ignore
+            builder.Append(buf) |> ignore
+        builder.ToString()
+    else
+        text
+
 let joinRoom (client: XmppClient) (roomJid: string) (nickname: string) (password: string option): unit =
     let room = bookmark roomJid nickname password
     client.BookmarkManager.Join room
@@ -59,7 +84,7 @@ let message (id: string) (toAddr: string) (text: string): XMPPMessage =
     m.SetAttributeValue(Type, "groupchat")
     m.SetAttributeValue(To, toAddr)
     let body = XElement(Body)
-    body.Value <- text
+    body.Value <- SanitizeXmlText text
     m.Add(body)
     m