perf(core/message_exporter): async download

- add retries

core/src/main/assets/web/export_template.html

@@ -232,15 +232,18 @@
     }
 
     function decodeMedia(element) {
-        const decodedData = new Uint8Array(
+        try {
-            inflate(
+            const decodedData = new Uint8Array(
-                base64decode(
+                inflate(
-                    element.innerHTML.substring(5, element.innerHTML.length - 4)
+                    base64decode(
+                        element.innerHTML.substring(5, element.innerHTML.length - 4)
+                    )
                 )
             )
-        )
+            return URL.createObjectURL(new Blob([decodedData]))
-
+        } catch (e) {
-        return URL.createObjectURL(new Blob([decodedData]))
+            return null
+        }
     }
 
     function makeMain() {

core/src/main/kotlin/me/rhunk/snapenhance/core/action/impl/ExportChatMessages.kt

@@ -5,18 +5,14 @@ import android.content.DialogInterface
 import android.os.Environment
 import android.text.InputType
 import android.widget.EditText
-import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.*
-import kotlinx.coroutines.Job
-import kotlinx.coroutines.joinAll
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.suspendCancellableCoroutine
 import me.rhunk.snapenhance.common.data.ContentType
 import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
 import me.rhunk.snapenhance.core.action.AbstractAction
 import me.rhunk.snapenhance.core.features.impl.messaging.Messaging
 import me.rhunk.snapenhance.core.logger.CoreLogger
+import me.rhunk.snapenhance.core.messaging.ConversationExporter
 import me.rhunk.snapenhance.core.messaging.ExportFormat
-import me.rhunk.snapenhance.core.messaging.MessageExporter
 import me.rhunk.snapenhance.core.ui.ViewAppearanceHelper
 import me.rhunk.snapenhance.core.wrapper.impl.Message
 import java.io.File
@@ -83,6 +79,7 @@ class ExportChatMessages : AbstractAction() {
         context.runOnUiThread {
             val mediasToDownload = mutableListOf<ContentType>()
             val contentTypes = arrayOf(
+                ContentType.CHAT,
                 ContentType.SNAP,
                 ContentType.EXTERNAL_MEDIA,
                 ContentType.NOTE,
@@ -142,25 +139,54 @@ class ExportChatMessages : AbstractAction() {
         }
     }
 
-    private suspend fun fetchMessagesPaginated(conversationId: String, lastMessageId: Long, amount: Int): List<Message> = suspendCancellableCoroutine { continuation ->
+    private suspend fun fetchMessagesPaginated(conversationId: String, lastMessageId: Long, amount: Int): List<Message> = runBlocking {
-        context.feature(Messaging::class).conversationManager?.fetchConversationWithMessagesPaginated(conversationId,
+        for (i in 0..5) {
-            lastMessageId,
+            val messages: List<Message>? = suspendCancellableCoroutine { continuation ->
-            amount, onSuccess = { messages ->
+                context.feature(Messaging::class).conversationManager?.fetchConversationWithMessagesPaginated(conversationId,
-                continuation.resumeWith(Result.success(messages))
+                    lastMessageId,
-            }, onError = {
+                    amount, onSuccess = { messages ->
-                continuation.resumeWith(Result.success(emptyList()))
+                        continuation.resumeWith(Result.success(messages))
-            }) ?: continuation.resumeWith(Result.success(emptyList()))
+                    }, onError = {
+                        continuation.resumeWith(Result.success(null))
+                    }) ?: continuation.resumeWith(Result.success(null))
+            }
+            if (messages != null) return@runBlocking messages
+            logDialog("Retrying in 1 second...")
+            delay(1000)
+        }
+        logDialog("Failed to fetch messages")
+        emptyList()
     }
 
     private suspend fun exportFullConversation(friendFeedEntry: FriendFeedEntry) {
         //first fetch the first message
         val conversationId = friendFeedEntry.key!!
         val conversationName = friendFeedEntry.feedDisplayName ?: friendFeedEntry.friendDisplayName!!.split("|").lastOrNull() ?: "unknown"
+        val conversationParticipants = context.database.getConversationParticipants(friendFeedEntry.key!!)
+            ?.mapNotNull {
+                context.database.getFriendInfo(it)
+            }?.associateBy { it.userId!! } ?: emptyMap()
+
+        val publicFolder = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "SnapEnhance")
+        val outputFile = publicFolder.resolve("conversation_${conversationName}_${System.currentTimeMillis()}.${exportType!!.extension}")
 
         logDialog(context.translation.format("chat_export.exporting_message", "conversation" to conversationName))
 
-        val foundMessages = fetchMessagesPaginated(conversationId, Long.MAX_VALUE, amount = 1).toMutableList()
+        val conversationExporter = ConversationExporter(
-        var lastMessageId = foundMessages.firstOrNull()?.messageDescriptor?.messageId ?: run {
+            context = context,
+            friendFeedEntry = friendFeedEntry,
+            conversationParticipants = conversationParticipants,
+            exportFormat = exportType!!,
+            messageTypeFilter = mediaToDownload,
+            cacheFolder = publicFolder.resolve("cache"),
+            outputFile = outputFile,
+        ).apply { init(); printLog = {
+            logDialog(it.toString())
+        } }
+
+        var foundMessageCount = 0
+
+        var lastMessageId = fetchMessagesPaginated(conversationId, Long.MAX_VALUE, amount = 1).firstOrNull()?.messageDescriptor?.messageId ?: run {
             logDialog(context.translation["chat_export.no_messages_found"])
             return
         }
@@ -168,40 +194,28 @@ class ExportChatMessages : AbstractAction() {
         while (true) {
             val fetchedMessages = fetchMessagesPaginated(conversationId, lastMessageId, amount = 500)
             if (fetchedMessages.isEmpty()) break
+            foundMessageCount += fetchedMessages.size
 
-            foundMessages.addAll(fetchedMessages)
+            if (amountOfMessages != null && foundMessageCount >= amountOfMessages!!) {
-            if (amountOfMessages != null && foundMessages.size >= amountOfMessages!!) {
+                fetchedMessages.subList(0, amountOfMessages!! - foundMessageCount).reversed().forEach { message ->
-                foundMessages.subList(amountOfMessages!!, foundMessages.size).clear()
+                    conversationExporter.readMessage(message)
+                }
                 break
             }
 
+            fetchedMessages.reversed().forEach { message ->
+                conversationExporter.readMessage(message)
+            }
+
             fetchedMessages.firstOrNull()?.let {
                 lastMessageId = it.messageDescriptor!!.messageId!!
             }
-            setStatus("Exporting (${foundMessages.size} / ${foundMessages.firstOrNull()?.orderKey})")
+            setStatus("Exporting (found ${foundMessageCount})")
         }
 
-        val outputFile = File(
+        if (exportType == ExportFormat.HTML) conversationExporter.awaitDownload()
-            Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS),
+        conversationExporter.close()
-            "SnapEnhance/conversation_${conversationName}_${System.currentTimeMillis()}.${exportType!!.extension}"
-        ).also { it.parentFile?.mkdirs() }
-
         logDialog(context.translation["chat_export.writing_output"])
-
-        runCatching {
-            MessageExporter(
-                context = context,
-                friendFeedEntry = friendFeedEntry,
-                outputFile = outputFile,
-                mediaToDownload = mediaToDownload,
-                printLog = ::logDialog
-            ).apply { readMessages(foundMessages) }.exportTo(exportType!!)
-        }.onFailure {
-            logDialog(context.translation.format("chat_export.export_failed","conversation" to it.message.toString()))
-            context.log.error("Failed to export conversation $conversationName", it)
-            return
-        }
-
         dialogLogs.clear()
         logDialog("\n" + context.translation.format("chat_export.exported_to",
             "path" to outputFile.absolutePath.toString()

core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/ConversationExporter.kt

@@ -0,0 +1,313 @@
+package me.rhunk.snapenhance.core.messaging
+
+import android.util.Base64InputStream
+import android.util.Base64OutputStream
+import com.google.gson.stream.JsonWriter
+import de.robv.android.xposed.XposedHelpers
+import me.rhunk.snapenhance.common.BuildConfig
+import me.rhunk.snapenhance.common.data.ContentType
+import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
+import me.rhunk.snapenhance.common.database.impl.FriendInfo
+import me.rhunk.snapenhance.common.util.snap.MediaDownloaderHelper
+import me.rhunk.snapenhance.common.util.snap.RemoteMediaResolver
+import me.rhunk.snapenhance.core.ModContext
+import me.rhunk.snapenhance.core.features.impl.downloader.decoder.AttachmentType
+import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder
+import me.rhunk.snapenhance.core.wrapper.impl.Message
+import me.rhunk.snapenhance.core.wrapper.impl.SnapUUID
+import java.io.BufferedInputStream
+import java.io.File
+import java.io.InputStream
+import java.io.OutputStream
+import java.text.DateFormat
+import java.util.Date
+import java.util.concurrent.Executors
+import java.util.zip.Deflater
+import java.util.zip.DeflaterInputStream
+import java.util.zip.DeflaterOutputStream
+import java.util.zip.ZipFile
+import kotlin.io.encoding.Base64
+import kotlin.io.encoding.ExperimentalEncodingApi
+
+@OptIn(ExperimentalEncodingApi::class)
+class ConversationExporter(
+    private val context: ModContext,
+    private val friendFeedEntry: FriendFeedEntry,
+    private val conversationParticipants: Map<String, FriendInfo>,
+    private val exportFormat: ExportFormat,
+    private val messageTypeFilter: List<ContentType>? = null,
+    private val cacheFolder: File,
+    private val outputFile: File
+) {
+    lateinit var printLog: (Any?) -> Unit
+
+    private val downloadThreadExecutor = Executors.newFixedThreadPool(4)
+    private val writeThreadExecutor = Executors.newSingleThreadExecutor()
+
+    private val conversationJsonDataFile by lazy { cacheFolder.resolve("messages.json") }
+    private val jsonDataWriter by lazy { JsonWriter(conversationJsonDataFile.writer()) }
+    private val outputFileStream by lazy { outputFile.outputStream() }
+    private val participants = mutableMapOf<String, Int>()
+
+    fun init() {
+        when (exportFormat) {
+            ExportFormat.TEXT -> {
+                outputFileStream.write("Conversation id: ${friendFeedEntry.key}\n".toByteArray())
+                outputFileStream.write("Conversation name: ${friendFeedEntry.feedDisplayName}\n".toByteArray())
+                outputFileStream.write("Participants:\n".toByteArray())
+                conversationParticipants.forEach { (userId, friendInfo) ->
+                    outputFileStream.write("  $userId: ${friendInfo.displayName}\n".toByteArray())
+                }
+                outputFileStream.write("\n\n".toByteArray())
+            }
+            else -> {
+                jsonDataWriter.isHtmlSafe = true
+                jsonDataWriter.serializeNulls = true
+
+                jsonDataWriter.beginObject()
+                jsonDataWriter.name("conversationId").value(friendFeedEntry.key)
+                jsonDataWriter.name("conversationName").value(friendFeedEntry.feedDisplayName)
+
+                var index = 0
+
+                jsonDataWriter.name("participants").apply {
+                    beginObject()
+                    conversationParticipants.forEach { (userId, friendInfo) ->
+                        jsonDataWriter.name(userId).beginObject()
+                        jsonDataWriter.name("id").value(index)
+                        jsonDataWriter.name("displayName").value(friendInfo.displayName)
+                        jsonDataWriter.name("username").value(friendInfo.usernameForSorting)
+                        jsonDataWriter.name("bitmojiSelfieId").value(friendInfo.bitmojiSelfieId)
+                        jsonDataWriter.endObject()
+                        participants[userId] = index++
+                    }
+                    endObject()
+                }
+
+                jsonDataWriter.name("messages").beginArray()
+
+                if (exportFormat != ExportFormat.HTML) return
+                outputFileStream.write("""
+                    <!DOCTYPE html>
+                    <html>
+                    <head>
+                        <meta charset="UTF-8">
+                        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+                        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+                        <title></title>
+                    </head>
+                """.trimIndent().toByteArray())
+
+                outputFileStream.write("<!-- This file was generated by SnapEnhance ${BuildConfig.VERSION_NAME} -->\n</head>".toByteArray())
+
+                outputFileStream.flush()
+            }
+        }
+    }
+
+
+    @OptIn(ExperimentalEncodingApi::class)
+    private fun downloadMedia(message: Message) {
+        downloadThreadExecutor.execute {
+            MessageDecoder.decode(message.messageContent!!).forEach decode@{ attachment ->
+                if (attachment.mediaUrlKey?.isEmpty() == true) return@decode
+                val protoMediaReference = Base64.UrlSafe.decode(attachment.mediaUrlKey ?: return@decode)
+
+                for (i in 0..5) {
+                    printLog("downloading ${attachment.mediaUrlKey}... (attempt ${i + 1}/5)")
+                    runCatching {
+                        RemoteMediaResolver.downloadBoltMedia(protoMediaReference, decryptionCallback = {
+                            (attachment.attachmentInfo?.encryption?.decryptInputStream(it) ?: it)
+                        }) { downloadedInputStream, _ ->
+                            downloadedInputStream.use { inputStream ->
+                                MediaDownloaderHelper.getSplitElements(inputStream) { type, splitInputStream ->
+                                    val mediaKey = "${type}_${Base64.UrlSafe.encode(protoMediaReference).replace("=", "")}"
+                                    val bufferedInputStream = BufferedInputStream(splitInputStream)
+                                    val fileType = MediaDownloaderHelper.getFileType(bufferedInputStream)
+                                    val mediaFile = cacheFolder.resolve("$mediaKey.${fileType.fileExtension}")
+
+                                    mediaFile.outputStream().use { fos ->
+                                        bufferedInputStream.copyTo(fos)
+                                    }
+
+                                    writeThreadExecutor.execute {
+                                        outputFileStream.write("<div class=\"media-$mediaKey\"><!-- ".toByteArray())
+                                        mediaFile.inputStream().use {
+                                            val deflateInputStream = DeflaterInputStream(it, Deflater(Deflater.BEST_SPEED, true))
+                                            (XposedHelpers.newInstance(
+                                                Base64InputStream::class.java,
+                                                deflateInputStream,
+                                                android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
+                                                true
+                                            ) as InputStream).copyTo(outputFileStream)
+                                            outputFileStream.write(" --></div>\n".toByteArray())
+                                            outputFileStream.flush()
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        return@decode
+                    }.onFailure {
+                        printLog("failed to download media ${attachment.mediaUrlKey}. retrying...")
+                        it.printStackTrace()
+                    }
+                }
+            }
+        }
+    }
+
+    fun readMessage(message: Message) {
+        if (exportFormat == ExportFormat.TEXT) {
+            val (displayName, senderUsername) = conversationParticipants[message.senderId.toString()]?.let {
+                it.displayName to it.mutableUsername
+            } ?: ("" to message.senderId.toString())
+
+            val date = DateFormat.getDateTimeInstance().format(Date(message.messageMetadata!!.createdAt ?: -1))
+            outputFileStream.write("[$date] - $displayName ($senderUsername): ${message.serialize() ?: message.messageContent?.contentType?.name}\n".toByteArray(Charsets.UTF_8))
+            return
+        }
+        val contentType = message.messageContent?.contentType ?: return
+
+        if (messageTypeFilter != null) {
+            if (!messageTypeFilter.contains(contentType))  return
+
+            if (contentType == ContentType.NOTE || contentType == ContentType.SNAP || contentType == ContentType.EXTERNAL_MEDIA) {
+                downloadMedia(message)
+            }
+        }
+
+
+        jsonDataWriter.apply {
+            beginObject()
+            name("orderKey").value(message.orderKey)
+            name("senderId").value(participants.getOrDefault(message.senderId.toString(), -1))
+            name("type").value(message.messageContent!!.contentType.toString())
+
+            fun addUUIDList(name: String, list: List<SnapUUID>) {
+                name(name).beginArray()
+                list.map { participants.getOrDefault(it.toString(), -1) }.forEach { value(it) }
+                endArray()
+            }
+
+            addUUIDList("savedBy", message.messageMetadata!!.savedBy!!)
+            addUUIDList("seenBy", message.messageMetadata!!.seenBy!!)
+            addUUIDList("openedBy", message.messageMetadata!!.openedBy!!)
+
+            name("reactions").beginObject()
+            message.messageMetadata!!.reactions!!.forEach { reaction ->
+                name(participants.getOrDefault(reaction.userId.toString(), -1L).toString()).value(reaction.reactionId)
+            }
+            endObject()
+
+            name("createdTimestamp").value(message.messageMetadata!!.createdAt)
+            name("readTimestamp").value(message.messageMetadata!!.readAt)
+            name("serializedContent").value(message.serialize())
+            name("rawContent").value(Base64.UrlSafe.encode(message.messageContent!!.content!!))
+            name("attachments").beginArray()
+            MessageDecoder.decode(message.messageContent!!)
+                .forEach attachments@{ attachments ->
+                    if (attachments.type == AttachmentType.STICKER) //TODO: implement stickers
+                        return@attachments
+                    beginObject()
+                    name("key").value(attachments.mediaUrlKey?.replace("=", ""))
+                    name("type").value(attachments.type.toString())
+                    name("encryption").apply {
+                        attachments.attachmentInfo?.encryption?.let { encryption ->
+                            beginObject()
+                            name("key").value(encryption.key)
+                            name("iv").value(encryption.iv)
+                            endObject()
+                        } ?: nullValue()
+                    }
+                    endObject()
+                }
+            endArray()
+            endObject()
+            flush()
+        }
+    }
+
+    fun awaitDownload() {
+        downloadThreadExecutor.shutdown()
+        downloadThreadExecutor.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
+        writeThreadExecutor.shutdown()
+        writeThreadExecutor.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
+    }
+
+    fun close() {
+        if (exportFormat != ExportFormat.TEXT) {
+            jsonDataWriter.endArray()
+            jsonDataWriter.endObject()
+            jsonDataWriter.flush()
+            jsonDataWriter.close()
+        }
+
+        if (exportFormat == ExportFormat.JSON) {
+            conversationJsonDataFile.inputStream().use {
+                it.copyTo(outputFileStream)
+            }
+        }
+
+        if (exportFormat == ExportFormat.HTML) {
+            //write the json file
+            outputFileStream.write("<script type=\"application/json\" class=\"exported_content\">".toByteArray())
+
+            (XposedHelpers.newInstance(
+                Base64OutputStream::class.java,
+                outputFileStream,
+                android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
+                true
+            ) as OutputStream).let { outputStream ->
+                val deflateOutputStream = DeflaterOutputStream(outputStream, Deflater(Deflater.BEST_COMPRESSION, true), true)
+                conversationJsonDataFile.inputStream().use {
+                    it.copyTo(deflateOutputStream)
+                }
+                deflateOutputStream.finish()
+                outputStream.flush()
+            }
+
+            outputFileStream.write("</script>\n".toByteArray())
+            printLog("writing template...")
+
+            runCatching {
+                ZipFile(context.bridgeClient.getApplicationApkPath()).use { apkFile ->
+                    //export rawinflate.js
+                    apkFile.getEntry("assets/web/rawinflate.js")?.let { entry ->
+                        outputFileStream.write("<script>".toByteArray())
+                        apkFile.getInputStream(entry).copyTo(outputFileStream)
+                        outputFileStream.write("</script>\n".toByteArray())
+                    }
+
+                    //export avenir next font
+                    apkFile.getEntry("assets/web/avenir_next_medium.ttf")?.let { entry ->
+                        val encodedFontData = Base64.Default.encode(apkFile.getInputStream(entry).readBytes())
+                        outputFileStream.write("""
+                            <style>
+                                @font-face {
+                                    font-family: 'Avenir Next';
+                                    src: url('data:font/truetype;charset=utf-8;base64, $encodedFontData');
+                                    font-weight: normal;
+                                    font-style: normal;
+                                }
+                            </style>
+                        """.trimIndent().toByteArray())
+                    }
+
+                    apkFile.getEntry("assets/web/export_template.html")?.let { entry ->
+                        apkFile.getInputStream(entry).copyTo(outputFileStream)
+                    }
+
+                    apkFile.close()
+                }
+            }.onFailure {
+                throw Throwable("Failed to read template from apk", it)
+            }
+
+            outputFileStream.write("</html>".toByteArray())
+        }
+
+        outputFileStream.flush()
+        outputFileStream.close()
+    }
+}

core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/ExportFormat.kt

@@ -0,0 +1,9 @@
+package me.rhunk.snapenhance.core.messaging;
+
+enum class ExportFormat(
+    val extension: String,
+){
+    JSON("json"),
+    TEXT("txt"),
+    HTML("html");
+}

core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/MessageExporter.kt

@@ -1,337 +0,0 @@
-package me.rhunk.snapenhance.core.messaging
-
-import android.os.Environment
-import android.util.Base64InputStream
-import android.util.Base64OutputStream
-import com.google.gson.JsonArray
-import com.google.gson.JsonNull
-import com.google.gson.JsonObject
-import de.robv.android.xposed.XposedHelpers
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.withContext
-import me.rhunk.snapenhance.common.BuildConfig
-import me.rhunk.snapenhance.common.data.ContentType
-import me.rhunk.snapenhance.common.data.FileType
-import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
-import me.rhunk.snapenhance.common.database.impl.FriendInfo
-import me.rhunk.snapenhance.common.util.protobuf.ProtoReader
-import me.rhunk.snapenhance.common.util.snap.MediaDownloaderHelper
-import me.rhunk.snapenhance.common.util.snap.RemoteMediaResolver
-import me.rhunk.snapenhance.core.ModContext
-import me.rhunk.snapenhance.core.features.impl.downloader.decoder.AttachmentType
-import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder
-import me.rhunk.snapenhance.core.wrapper.impl.Message
-import me.rhunk.snapenhance.core.wrapper.impl.SnapUUID
-import java.io.BufferedInputStream
-import java.io.File
-import java.io.FileOutputStream
-import java.io.InputStream
-import java.io.OutputStream
-import java.text.SimpleDateFormat
-import java.util.Collections
-import java.util.Date
-import java.util.Locale
-import java.util.concurrent.Executors
-import java.util.concurrent.TimeUnit
-import java.util.zip.Deflater
-import java.util.zip.DeflaterInputStream
-import java.util.zip.DeflaterOutputStream
-import java.util.zip.ZipFile
-import kotlin.io.encoding.Base64
-import kotlin.io.encoding.ExperimentalEncodingApi
-
-
-enum class ExportFormat(
-    val extension: String,
-){
-    JSON("json"),
-    TEXT("txt"),
-    HTML("html");
-}
-
-@OptIn(ExperimentalEncodingApi::class)
-class MessageExporter(
-    private val context: ModContext,
-    private val outputFile: File,
-    private val friendFeedEntry: FriendFeedEntry,
-    private val mediaToDownload: List<ContentType>? = null,
-    private val printLog: (String) -> Unit = {},
-) {
-    private lateinit var conversationParticipants: Map<String, FriendInfo>
-    private lateinit var messages: List<Message>
-
-    fun readMessages(messages: List<Message>) {
-        conversationParticipants =
-            context.database.getConversationParticipants(friendFeedEntry.key!!)
-                ?.mapNotNull {
-                    context.database.getFriendInfo(it)
-                }?.associateBy { it.userId!! } ?: emptyMap()
-
-        if (conversationParticipants.isEmpty())
-            throw Throwable("Failed to get conversation participants for ${friendFeedEntry.key}")
-
-        this.messages = messages.sortedBy { it.orderKey }
-    }
-
-    private fun serializeMessageContent(message: Message): String? {
-        return if (message.messageContent!!.contentType == ContentType.CHAT) {
-            ProtoReader(message.messageContent!!.content!!).getString(2, 1) ?: "Failed to parse message"
-        } else null
-    }
-
-    private fun exportText(output: OutputStream) {
-        val writer = output.bufferedWriter()
-        writer.write("Conversation key: ${friendFeedEntry.key}\n")
-        writer.write("Conversation Name: ${friendFeedEntry.feedDisplayName}\n")
-        writer.write("Participants:\n")
-        conversationParticipants.forEach { (userId, friendInfo) ->
-            writer.write("  $userId: ${friendInfo.displayName}\n")
-        }
-
-        writer.write("\nMessages:\n")
-        messages.forEach { message ->
-            val sender = conversationParticipants[message.senderId.toString()]
-            val senderUsername = sender?.usernameForSorting ?: message.senderId.toString()
-            val senderDisplayName = sender?.displayName ?: message.senderId.toString()
-            val messageContent = serializeMessageContent(message) ?: message.messageContent!!.contentType?.name
-            val date = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH).format(Date(message.messageMetadata!!.createdAt!!))
-            writer.write("[$date] - $senderDisplayName (${senderUsername}): $messageContent\n")
-        }
-        writer.flush()
-    }
-
-    private suspend fun exportHtml(output: OutputStream) {
-        val downloadMediaCacheFolder = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "SnapEnhance/cache").also { it.mkdirs() }
-        val mediaFiles = Collections.synchronizedMap(mutableMapOf<String, Pair<FileType, File>>())
-        val threadPool = Executors.newFixedThreadPool(15)
-
-        withContext(Dispatchers.IO) {
-            var processCount = 0
-
-            fun updateProgress(type: String) {
-                val total = messages.filter {
-                    mediaToDownload?.contains(it.messageContent!!.contentType) ?: false
-                }.size
-                processCount++
-                printLog("$type $processCount/$total")
-            }
-
-            messages.filter {
-                mediaToDownload?.contains(it.messageContent!!.contentType) ?: false
-            }.forEach { message ->
-                threadPool.execute {
-                    MessageDecoder.decode(message.messageContent!!).forEach decode@{ attachment ->
-                        val protoMediaReference = Base64.UrlSafe.decode(attachment.mediaUrlKey ?: return@decode)
-
-                        runCatching {
-                            RemoteMediaResolver.downloadBoltMedia(protoMediaReference, decryptionCallback = {
-                                (attachment.attachmentInfo?.encryption?.decryptInputStream(it) ?: it)
-                            }) { downloadedInputStream, _ ->
-                                downloadedInputStream.use { inputStream ->
-                                    MediaDownloaderHelper.getSplitElements(inputStream) { type, splitInputStream ->
-                                        val fileName = "${type}_${Base64.UrlSafe.encode(protoMediaReference).replace("=", "")}"
-                                        val bufferedInputStream = BufferedInputStream(splitInputStream)
-                                        val fileType = MediaDownloaderHelper.getFileType(bufferedInputStream)
-                                        val mediaFile = File(downloadMediaCacheFolder, "$fileName.${fileType.fileExtension}")
-
-                                        FileOutputStream(mediaFile).use { fos ->
-                                            bufferedInputStream.copyTo(fos)
-                                        }
-
-                                        mediaFiles[fileName] = fileType to mediaFile
-                                    }
-                                }
-                            }
-
-                            updateProgress("downloaded")
-                        }.onFailure {
-                            printLog("failed to download media for ${message.messageDescriptor!!.conversationId}_${message.orderKey}")
-                            context.log.error("failed to download media for ${message.messageDescriptor!!.conversationId}_${message.orderKey}", it)
-                        }
-                    }
-                }
-            }
-
-            threadPool.shutdown()
-            threadPool.awaitTermination(30, TimeUnit.DAYS)
-            processCount = 0
-
-            printLog("writing downloaded medias...")
-
-            //write the head of the html file
-            output.write("""
-                <!DOCTYPE html>
-                <html>
-                <head>
-                    <meta charset="UTF-8">
-                    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-                    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-                    <title></title>
-                </head>
-            """.trimIndent().toByteArray())
-
-            output.write("<!-- This file was generated by SnapEnhance ${BuildConfig.VERSION_NAME} -->\n".toByteArray())
-
-            mediaFiles.forEach { (key, filePair) ->
-                output.write("<div class=\"media-$key\"><!-- ".toByteArray())
-                filePair.second.inputStream().use { inputStream ->
-                    val deflateInputStream = DeflaterInputStream(inputStream, Deflater(Deflater.BEST_COMPRESSION, true))
-                    (XposedHelpers.newInstance(
-                        Base64InputStream::class.java,
-                        deflateInputStream,
-                        android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
-                        true
-                    ) as InputStream).copyTo(output)
-                }
-                output.write(" --></div>\n".toByteArray())
-                output.flush()
-                updateProgress("wrote")
-            }
-            printLog("writing json conversation data...")
-
-            //write the json file
-            output.write("<script type=\"application/json\" class=\"exported_content\">".toByteArray())
-
-            val b64os = (XposedHelpers.newInstance(
-                Base64OutputStream::class.java,
-                output,
-                android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
-                true
-            ) as OutputStream)
-            val deflateOutputStream = DeflaterOutputStream(b64os, Deflater(Deflater.BEST_COMPRESSION, true), true)
-            exportJson(deflateOutputStream)
-            deflateOutputStream.finish()
-            b64os.flush()
-
-            output.write("</script>\n".toByteArray())
-
-            printLog("writing template...")
-
-            runCatching {
-                ZipFile(context.bridgeClient.getApplicationApkPath()).use { apkFile ->
-                    //export rawinflate.js
-                    apkFile.getEntry("assets/web/rawinflate.js")?.let { entry ->
-                        output.write("<script>".toByteArray())
-                        apkFile.getInputStream(entry).copyTo(output)
-                        output.write("</script>\n".toByteArray())
-                    }
-
-                    //export avenir next font
-                    apkFile.getEntry("assets/web/avenir_next_medium.ttf")?.let { entry ->
-                        val encodedFontData = Base64.Default.encode(apkFile.getInputStream(entry).readBytes())
-                        output.write("""
-                            <style>
-                                @font-face {
-                                    font-family: 'Avenir Next';
-                                    src: url('data:font/truetype;charset=utf-8;base64, $encodedFontData');
-                                    font-weight: normal;
-                                    font-style: normal;
-                                }
-                            </style>
-                        """.trimIndent().toByteArray())
-                    }
-
-                    apkFile.getEntry("assets/web/export_template.html")?.let { entry ->
-                        apkFile.getInputStream(entry).copyTo(output)
-                    }
-
-                    apkFile.close()
-                }
-            }.onFailure {
-                throw Throwable("Failed to read template from apk", it)
-            }
-
-            output.write("</html>".toByteArray())
-            output.close()
-        }
-    }
-
-    private fun exportJson(output: OutputStream) {
-        val rootObject = JsonObject().apply {
-            addProperty("conversationId", friendFeedEntry.key)
-            addProperty("conversationName", friendFeedEntry.feedDisplayName)
-
-            var index = 0
-            val participants = mutableMapOf<String, Int>()
-
-            add("participants", JsonObject().apply {
-                conversationParticipants.forEach { (userId, friendInfo) ->
-                    add(userId, JsonObject().apply {
-                        addProperty("id", index)
-                        addProperty("displayName", friendInfo.displayName)
-                        addProperty("username", friendInfo.usernameForSorting)
-                        addProperty("bitmojiSelfieId", friendInfo.bitmojiSelfieId)
-                    })
-                    participants[userId] = index++
-                }
-            })
-            add("messages", JsonArray().apply {
-                messages.forEach { message ->
-                    add(JsonObject().apply {
-                        addProperty("orderKey", message.orderKey)
-                        addProperty("senderId", participants.getOrDefault(message.senderId.toString(), -1))
-                        addProperty("type", message.messageContent!!.contentType.toString())
-
-                        fun addUUIDList(name: String, list: List<SnapUUID>) {
-                            add(name, JsonArray().apply {
-                                list.map { participants.getOrDefault(it.toString(), -1) }.forEach { add(it) }
-                            })
-                        }
-
-                        addUUIDList("savedBy", message.messageMetadata!!.savedBy!!)
-                        addUUIDList("seenBy", message.messageMetadata!!.seenBy!!)
-                        addUUIDList("openedBy", message.messageMetadata!!.openedBy!!)
-
-                        add("reactions", JsonObject().apply {
-                            message.messageMetadata!!.reactions!!.forEach { reaction ->
-                                addProperty(
-                                    participants.getOrDefault(reaction.userId.toString(), -1L).toString(),
-                                    reaction.reactionId
-                                )
-                            }
-                        })
-
-                        addProperty("createdTimestamp", message.messageMetadata!!.createdAt)
-                        addProperty("readTimestamp", message.messageMetadata!!.readAt)
-                        addProperty("serializedContent", serializeMessageContent(message))
-                        addProperty("rawContent", Base64.UrlSafe.encode(message.messageContent!!.content!!))
-
-                        add("attachments", JsonArray().apply {
-                            MessageDecoder.decode(message.messageContent!!)
-                                .forEach attachments@{ attachments ->
-                                if (attachments.type == AttachmentType.STICKER) //TODO: implement stickers
-                                    return@attachments
-                                add(JsonObject().apply {
-                                    addProperty("key", attachments.mediaUrlKey?.replace("=", ""))
-                                    addProperty("type", attachments.type.toString())
-                                    add("encryption", attachments.attachmentInfo?.encryption?.let { encryption ->
-                                        JsonObject().apply {
-                                            addProperty("key", encryption.key)
-                                            addProperty("iv", encryption.iv)
-                                        }
-                                    } ?: JsonNull.INSTANCE)
-                                })
-                            }
-                        })
-                    })
-                }
-            })
-        }
-
-        output.write(context.gson.toJson(rootObject).toByteArray())
-        output.flush()
-    }
-
-    suspend fun exportTo(exportFormat: ExportFormat) {
-        withContext(Dispatchers.IO) {
-            FileOutputStream(outputFile).apply {
-                when (exportFormat) {
-                    ExportFormat.HTML -> exportHtml(this)
-                    ExportFormat.JSON -> exportJson(this)
-                    ExportFormat.TEXT -> exportText(this)
-                }
-                close()
-            }
-        }
-    }
-}

core/src/main/kotlin/me/rhunk/snapenhance/core/wrapper/impl/ConversationManager.kt

@@ -3,6 +3,7 @@ package me.rhunk.snapenhance.core.wrapper.impl
 import me.rhunk.snapenhance.common.data.MessageUpdate
 import me.rhunk.snapenhance.core.ModContext
 import me.rhunk.snapenhance.core.util.CallbackBuilder
+import me.rhunk.snapenhance.core.util.ktx.getObjectField
 import me.rhunk.snapenhance.core.util.ktx.setObjectField
 import me.rhunk.snapenhance.core.wrapper.AbstractWrapper
 
@@ -18,6 +19,7 @@ class ConversationManager(
     private val fetchConversationWithMessagesPaginatedMethod by lazy { findMethodByName("fetchConversationWithMessagesPaginated") }
     private val fetchConversationWithMessagesMethod by lazy { findMethodByName("fetchConversationWithMessages") }
     private val fetchMessageByServerId by lazy { findMethodByName("fetchMessageByServerId") }
+    private val fetchMessagesByServerIds by lazy { findMethodByName("fetchMessagesByServerIds") }
     private val displayedMessagesMethod by lazy { findMethodByName("displayedMessages") }
     private val fetchMessage by lazy { findMethodByName("fetchMessage") }
 
@@ -105,4 +107,25 @@ class ConversationManager(
                 }.build()
         )
     }
+
+    fun fetchMessagesByServerIds(conversationId: String, serverMessageIds: List<Long>, onSuccess: (List<Message>) -> Unit, onError: (error: String) -> Unit) {
+        fetchMessagesByServerIds.invoke(
+            instanceNonNull(),
+            serverMessageIds.map {
+                CallbackBuilder.createEmptyObject(context.classCache.serverMessageIdentifier.constructors.first())?.apply {
+                    setObjectField("mServerConversationId", conversationId.toSnapUUID().instanceNonNull())
+                    setObjectField("mServerMessageId", it)
+                }
+            },
+            CallbackBuilder(context.mappings.getMappedClass("callbacks", "FetchMessagesByServerIdsCallback"))
+                .override("onSuccess") { param ->
+                    onSuccess(param.arg<List<*>>(0).mapNotNull {
+                        Message(it?.getObjectField("mMessage") ?: return@mapNotNull null)
+                    })
+                }
+                .override("onError") {
+                    onError(it.arg<Any>(0).toString())
+                }.build()
+        )
+    }
 }

core/src/main/kotlin/me/rhunk/snapenhance/core/wrapper/impl/Message.kt

@@ -1,6 +1,8 @@
 package me.rhunk.snapenhance.core.wrapper.impl
 
+import me.rhunk.snapenhance.common.data.ContentType
 import me.rhunk.snapenhance.common.data.MessageState
+import me.rhunk.snapenhance.common.util.protobuf.ProtoReader
 import me.rhunk.snapenhance.core.wrapper.AbstractWrapper
 import org.mozilla.javascript.annotations.JSGetter
 import org.mozilla.javascript.annotations.JSSetter
@@ -18,4 +20,8 @@ class Message(obj: Any?) : AbstractWrapper(obj) {
     var messageMetadata by field("mMetadata") { MessageMetadata(it) }
     @get:JSGetter @set:JSSetter
     var messageState by enum("mState", MessageState.COMMITTED)
+
+    fun serialize() = if (messageContent!!.contentType == ContentType.CHAT) {
+        ProtoReader(messageContent!!.content!!).getString(2, 1) ?: "Failed to parse message"
+    } else null
 }