mirror of
https://github.com/rhunk/SnapEnhance.git
synced 2025-06-12 13:17:42 +02:00
perf(core/message_exporter): async download
- add retries
This commit is contained in:
rhunk
@ -232,15 +232,18 @@
|
||||
}
|
||||
|
||||
function decodeMedia(element) {
|
||||
const decodedData = new Uint8Array(
|
||||
inflate(
|
||||
base64decode(
|
||||
element.innerHTML.substring(5, element.innerHTML.length - 4)
|
||||
try {
|
||||
const decodedData = new Uint8Array(
|
||||
inflate(
|
||||
base64decode(
|
||||
element.innerHTML.substring(5, element.innerHTML.length - 4)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return URL.createObjectURL(new Blob([decodedData]))
|
||||
return URL.createObjectURL(new Blob([decodedData]))
|
||||
} catch (e) {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function makeMain() {
|
||||
|
||||
@ -5,18 +5,14 @@ import android.content.DialogInterface
|
||||
import android.os.Environment
|
||||
import android.text.InputType
|
||||
import android.widget.EditText
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.joinAll
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlinx.coroutines.*
|
||||
import me.rhunk.snapenhance.common.data.ContentType
|
||||
import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
|
||||
import me.rhunk.snapenhance.core.action.AbstractAction
|
||||
import me.rhunk.snapenhance.core.features.impl.messaging.Messaging
|
||||
import me.rhunk.snapenhance.core.logger.CoreLogger
|
||||
import me.rhunk.snapenhance.core.messaging.ConversationExporter
|
||||
import me.rhunk.snapenhance.core.messaging.ExportFormat
|
||||
import me.rhunk.snapenhance.core.messaging.MessageExporter
|
||||
import me.rhunk.snapenhance.core.ui.ViewAppearanceHelper
|
||||
import me.rhunk.snapenhance.core.wrapper.impl.Message
|
||||
import java.io.File
|
||||
@ -83,6 +79,7 @@ class ExportChatMessages : AbstractAction() {
|
||||
context.runOnUiThread {
|
||||
val mediasToDownload = mutableListOf<ContentType>()
|
||||
val contentTypes = arrayOf(
|
||||
ContentType.CHAT,
|
||||
ContentType.SNAP,
|
||||
ContentType.EXTERNAL_MEDIA,
|
||||
ContentType.NOTE,
|
||||
@ -142,25 +139,54 @@ class ExportChatMessages : AbstractAction() {
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun fetchMessagesPaginated(conversationId: String, lastMessageId: Long, amount: Int): List<Message> = suspendCancellableCoroutine { continuation ->
|
||||
context.feature(Messaging::class).conversationManager?.fetchConversationWithMessagesPaginated(conversationId,
|
||||
lastMessageId,
|
||||
amount, onSuccess = { messages ->
|
||||
continuation.resumeWith(Result.success(messages))
|
||||
}, onError = {
|
||||
continuation.resumeWith(Result.success(emptyList()))
|
||||
}) ?: continuation.resumeWith(Result.success(emptyList()))
|
||||
private suspend fun fetchMessagesPaginated(conversationId: String, lastMessageId: Long, amount: Int): List<Message> = runBlocking {
|
||||
for (i in 0..5) {
|
||||
val messages: List<Message>? = suspendCancellableCoroutine { continuation ->
|
||||
context.feature(Messaging::class).conversationManager?.fetchConversationWithMessagesPaginated(conversationId,
|
||||
lastMessageId,
|
||||
amount, onSuccess = { messages ->
|
||||
continuation.resumeWith(Result.success(messages))
|
||||
}, onError = {
|
||||
continuation.resumeWith(Result.success(null))
|
||||
}) ?: continuation.resumeWith(Result.success(null))
|
||||
}
|
||||
if (messages != null) return@runBlocking messages
|
||||
logDialog("Retrying in 1 second...")
|
||||
delay(1000)
|
||||
}
|
||||
logDialog("Failed to fetch messages")
|
||||
emptyList()
|
||||
}
|
||||
|
||||
private suspend fun exportFullConversation(friendFeedEntry: FriendFeedEntry) {
|
||||
//first fetch the first message
|
||||
val conversationId = friendFeedEntry.key!!
|
||||
val conversationName = friendFeedEntry.feedDisplayName ?: friendFeedEntry.friendDisplayName!!.split("|").lastOrNull() ?: "unknown"
|
||||
val conversationParticipants = context.database.getConversationParticipants(friendFeedEntry.key!!)
|
||||
?.mapNotNull {
|
||||
context.database.getFriendInfo(it)
|
||||
}?.associateBy { it.userId!! } ?: emptyMap()
|
||||
|
||||
val publicFolder = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "SnapEnhance")
|
||||
val outputFile = publicFolder.resolve("conversation_${conversationName}_${System.currentTimeMillis()}.${exportType!!.extension}")
|
||||
|
||||
logDialog(context.translation.format("chat_export.exporting_message", "conversation" to conversationName))
|
||||
|
||||
val foundMessages = fetchMessagesPaginated(conversationId, Long.MAX_VALUE, amount = 1).toMutableList()
|
||||
var lastMessageId = foundMessages.firstOrNull()?.messageDescriptor?.messageId ?: run {
|
||||
val conversationExporter = ConversationExporter(
|
||||
context = context,
|
||||
friendFeedEntry = friendFeedEntry,
|
||||
conversationParticipants = conversationParticipants,
|
||||
exportFormat = exportType!!,
|
||||
messageTypeFilter = mediaToDownload,
|
||||
cacheFolder = publicFolder.resolve("cache"),
|
||||
outputFile = outputFile,
|
||||
).apply { init(); printLog = {
|
||||
logDialog(it.toString())
|
||||
} }
|
||||
|
||||
var foundMessageCount = 0
|
||||
|
||||
var lastMessageId = fetchMessagesPaginated(conversationId, Long.MAX_VALUE, amount = 1).firstOrNull()?.messageDescriptor?.messageId ?: run {
|
||||
logDialog(context.translation["chat_export.no_messages_found"])
|
||||
return
|
||||
}
|
||||
@ -168,40 +194,28 @@ class ExportChatMessages : AbstractAction() {
|
||||
while (true) {
|
||||
val fetchedMessages = fetchMessagesPaginated(conversationId, lastMessageId, amount = 500)
|
||||
if (fetchedMessages.isEmpty()) break
|
||||
foundMessageCount += fetchedMessages.size
|
||||
|
||||
foundMessages.addAll(fetchedMessages)
|
||||
if (amountOfMessages != null && foundMessages.size >= amountOfMessages!!) {
|
||||
foundMessages.subList(amountOfMessages!!, foundMessages.size).clear()
|
||||
if (amountOfMessages != null && foundMessageCount >= amountOfMessages!!) {
|
||||
fetchedMessages.subList(0, amountOfMessages!! - foundMessageCount).reversed().forEach { message ->
|
||||
conversationExporter.readMessage(message)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
fetchedMessages.reversed().forEach { message ->
|
||||
conversationExporter.readMessage(message)
|
||||
}
|
||||
|
||||
fetchedMessages.firstOrNull()?.let {
|
||||
lastMessageId = it.messageDescriptor!!.messageId!!
|
||||
}
|
||||
setStatus("Exporting (${foundMessages.size} / ${foundMessages.firstOrNull()?.orderKey})")
|
||||
setStatus("Exporting (found ${foundMessageCount})")
|
||||
}
|
||||
|
||||
val outputFile = File(
|
||||
Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS),
|
||||
"SnapEnhance/conversation_${conversationName}_${System.currentTimeMillis()}.${exportType!!.extension}"
|
||||
).also { it.parentFile?.mkdirs() }
|
||||
|
||||
if (exportType == ExportFormat.HTML) conversationExporter.awaitDownload()
|
||||
conversationExporter.close()
|
||||
logDialog(context.translation["chat_export.writing_output"])
|
||||
|
||||
runCatching {
|
||||
MessageExporter(
|
||||
context = context,
|
||||
friendFeedEntry = friendFeedEntry,
|
||||
outputFile = outputFile,
|
||||
mediaToDownload = mediaToDownload,
|
||||
printLog = ::logDialog
|
||||
).apply { readMessages(foundMessages) }.exportTo(exportType!!)
|
||||
}.onFailure {
|
||||
logDialog(context.translation.format("chat_export.export_failed","conversation" to it.message.toString()))
|
||||
context.log.error("Failed to export conversation $conversationName", it)
|
||||
return
|
||||
}
|
||||
|
||||
dialogLogs.clear()
|
||||
logDialog("\n" + context.translation.format("chat_export.exported_to",
|
||||
"path" to outputFile.absolutePath.toString()
|
||||
|
||||
@ -0,0 +1,313 @@
|
||||
package me.rhunk.snapenhance.core.messaging
|
||||
|
||||
import android.util.Base64InputStream
|
||||
import android.util.Base64OutputStream
|
||||
import com.google.gson.stream.JsonWriter
|
||||
import de.robv.android.xposed.XposedHelpers
|
||||
import me.rhunk.snapenhance.common.BuildConfig
|
||||
import me.rhunk.snapenhance.common.data.ContentType
|
||||
import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
|
||||
import me.rhunk.snapenhance.common.database.impl.FriendInfo
|
||||
import me.rhunk.snapenhance.common.util.snap.MediaDownloaderHelper
|
||||
import me.rhunk.snapenhance.common.util.snap.RemoteMediaResolver
|
||||
import me.rhunk.snapenhance.core.ModContext
|
||||
import me.rhunk.snapenhance.core.features.impl.downloader.decoder.AttachmentType
|
||||
import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder
|
||||
import me.rhunk.snapenhance.core.wrapper.impl.Message
|
||||
import me.rhunk.snapenhance.core.wrapper.impl.SnapUUID
|
||||
import java.io.BufferedInputStream
|
||||
import java.io.File
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import java.text.DateFormat
|
||||
import java.util.Date
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.zip.Deflater
|
||||
import java.util.zip.DeflaterInputStream
|
||||
import java.util.zip.DeflaterOutputStream
|
||||
import java.util.zip.ZipFile
|
||||
import kotlin.io.encoding.Base64
|
||||
import kotlin.io.encoding.ExperimentalEncodingApi
|
||||
|
||||
@OptIn(ExperimentalEncodingApi::class)
|
||||
class ConversationExporter(
|
||||
private val context: ModContext,
|
||||
private val friendFeedEntry: FriendFeedEntry,
|
||||
private val conversationParticipants: Map<String, FriendInfo>,
|
||||
private val exportFormat: ExportFormat,
|
||||
private val messageTypeFilter: List<ContentType>? = null,
|
||||
private val cacheFolder: File,
|
||||
private val outputFile: File
|
||||
) {
|
||||
lateinit var printLog: (Any?) -> Unit
|
||||
|
||||
private val downloadThreadExecutor = Executors.newFixedThreadPool(4)
|
||||
private val writeThreadExecutor = Executors.newSingleThreadExecutor()
|
||||
|
||||
private val conversationJsonDataFile by lazy { cacheFolder.resolve("messages.json") }
|
||||
private val jsonDataWriter by lazy { JsonWriter(conversationJsonDataFile.writer()) }
|
||||
private val outputFileStream by lazy { outputFile.outputStream() }
|
||||
private val participants = mutableMapOf<String, Int>()
|
||||
|
||||
fun init() {
|
||||
when (exportFormat) {
|
||||
ExportFormat.TEXT -> {
|
||||
outputFileStream.write("Conversation id: ${friendFeedEntry.key}\n".toByteArray())
|
||||
outputFileStream.write("Conversation name: ${friendFeedEntry.feedDisplayName}\n".toByteArray())
|
||||
outputFileStream.write("Participants:\n".toByteArray())
|
||||
conversationParticipants.forEach { (userId, friendInfo) ->
|
||||
outputFileStream.write(" $userId: ${friendInfo.displayName}\n".toByteArray())
|
||||
}
|
||||
outputFileStream.write("\n\n".toByteArray())
|
||||
}
|
||||
else -> {
|
||||
jsonDataWriter.isHtmlSafe = true
|
||||
jsonDataWriter.serializeNulls = true
|
||||
|
||||
jsonDataWriter.beginObject()
|
||||
jsonDataWriter.name("conversationId").value(friendFeedEntry.key)
|
||||
jsonDataWriter.name("conversationName").value(friendFeedEntry.feedDisplayName)
|
||||
|
||||
var index = 0
|
||||
|
||||
jsonDataWriter.name("participants").apply {
|
||||
beginObject()
|
||||
conversationParticipants.forEach { (userId, friendInfo) ->
|
||||
jsonDataWriter.name(userId).beginObject()
|
||||
jsonDataWriter.name("id").value(index)
|
||||
jsonDataWriter.name("displayName").value(friendInfo.displayName)
|
||||
jsonDataWriter.name("username").value(friendInfo.usernameForSorting)
|
||||
jsonDataWriter.name("bitmojiSelfieId").value(friendInfo.bitmojiSelfieId)
|
||||
jsonDataWriter.endObject()
|
||||
participants[userId] = index++
|
||||
}
|
||||
endObject()
|
||||
}
|
||||
|
||||
jsonDataWriter.name("messages").beginArray()
|
||||
|
||||
if (exportFormat != ExportFormat.HTML) return
|
||||
outputFileStream.write("""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title></title>
|
||||
</head>
|
||||
""".trimIndent().toByteArray())
|
||||
|
||||
outputFileStream.write("<!-- This file was generated by SnapEnhance ${BuildConfig.VERSION_NAME} -->\n</head>".toByteArray())
|
||||
|
||||
outputFileStream.flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@OptIn(ExperimentalEncodingApi::class)
|
||||
private fun downloadMedia(message: Message) {
|
||||
downloadThreadExecutor.execute {
|
||||
MessageDecoder.decode(message.messageContent!!).forEach decode@{ attachment ->
|
||||
if (attachment.mediaUrlKey?.isEmpty() == true) return@decode
|
||||
val protoMediaReference = Base64.UrlSafe.decode(attachment.mediaUrlKey ?: return@decode)
|
||||
|
||||
for (i in 0..5) {
|
||||
printLog("downloading ${attachment.mediaUrlKey}... (attempt ${i + 1}/5)")
|
||||
runCatching {
|
||||
RemoteMediaResolver.downloadBoltMedia(protoMediaReference, decryptionCallback = {
|
||||
(attachment.attachmentInfo?.encryption?.decryptInputStream(it) ?: it)
|
||||
}) { downloadedInputStream, _ ->
|
||||
downloadedInputStream.use { inputStream ->
|
||||
MediaDownloaderHelper.getSplitElements(inputStream) { type, splitInputStream ->
|
||||
val mediaKey = "${type}_${Base64.UrlSafe.encode(protoMediaReference).replace("=", "")}"
|
||||
val bufferedInputStream = BufferedInputStream(splitInputStream)
|
||||
val fileType = MediaDownloaderHelper.getFileType(bufferedInputStream)
|
||||
val mediaFile = cacheFolder.resolve("$mediaKey.${fileType.fileExtension}")
|
||||
|
||||
mediaFile.outputStream().use { fos ->
|
||||
bufferedInputStream.copyTo(fos)
|
||||
}
|
||||
|
||||
writeThreadExecutor.execute {
|
||||
outputFileStream.write("<div class=\"media-$mediaKey\"><!-- ".toByteArray())
|
||||
mediaFile.inputStream().use {
|
||||
val deflateInputStream = DeflaterInputStream(it, Deflater(Deflater.BEST_SPEED, true))
|
||||
(XposedHelpers.newInstance(
|
||||
Base64InputStream::class.java,
|
||||
deflateInputStream,
|
||||
android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
|
||||
true
|
||||
) as InputStream).copyTo(outputFileStream)
|
||||
outputFileStream.write(" --></div>\n".toByteArray())
|
||||
outputFileStream.flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return@decode
|
||||
}.onFailure {
|
||||
printLog("failed to download media ${attachment.mediaUrlKey}. retrying...")
|
||||
it.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun readMessage(message: Message) {
|
||||
if (exportFormat == ExportFormat.TEXT) {
|
||||
val (displayName, senderUsername) = conversationParticipants[message.senderId.toString()]?.let {
|
||||
it.displayName to it.mutableUsername
|
||||
} ?: ("" to message.senderId.toString())
|
||||
|
||||
val date = DateFormat.getDateTimeInstance().format(Date(message.messageMetadata!!.createdAt ?: -1))
|
||||
outputFileStream.write("[$date] - $displayName ($senderUsername): ${message.serialize() ?: message.messageContent?.contentType?.name}\n".toByteArray(Charsets.UTF_8))
|
||||
return
|
||||
}
|
||||
val contentType = message.messageContent?.contentType ?: return
|
||||
|
||||
if (messageTypeFilter != null) {
|
||||
if (!messageTypeFilter.contains(contentType)) return
|
||||
|
||||
if (contentType == ContentType.NOTE || contentType == ContentType.SNAP || contentType == ContentType.EXTERNAL_MEDIA) {
|
||||
downloadMedia(message)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
jsonDataWriter.apply {
|
||||
beginObject()
|
||||
name("orderKey").value(message.orderKey)
|
||||
name("senderId").value(participants.getOrDefault(message.senderId.toString(), -1))
|
||||
name("type").value(message.messageContent!!.contentType.toString())
|
||||
|
||||
fun addUUIDList(name: String, list: List<SnapUUID>) {
|
||||
name(name).beginArray()
|
||||
list.map { participants.getOrDefault(it.toString(), -1) }.forEach { value(it) }
|
||||
endArray()
|
||||
}
|
||||
|
||||
addUUIDList("savedBy", message.messageMetadata!!.savedBy!!)
|
||||
addUUIDList("seenBy", message.messageMetadata!!.seenBy!!)
|
||||
addUUIDList("openedBy", message.messageMetadata!!.openedBy!!)
|
||||
|
||||
name("reactions").beginObject()
|
||||
message.messageMetadata!!.reactions!!.forEach { reaction ->
|
||||
name(participants.getOrDefault(reaction.userId.toString(), -1L).toString()).value(reaction.reactionId)
|
||||
}
|
||||
endObject()
|
||||
|
||||
name("createdTimestamp").value(message.messageMetadata!!.createdAt)
|
||||
name("readTimestamp").value(message.messageMetadata!!.readAt)
|
||||
name("serializedContent").value(message.serialize())
|
||||
name("rawContent").value(Base64.UrlSafe.encode(message.messageContent!!.content!!))
|
||||
name("attachments").beginArray()
|
||||
MessageDecoder.decode(message.messageContent!!)
|
||||
.forEach attachments@{ attachments ->
|
||||
if (attachments.type == AttachmentType.STICKER) //TODO: implement stickers
|
||||
return@attachments
|
||||
beginObject()
|
||||
name("key").value(attachments.mediaUrlKey?.replace("=", ""))
|
||||
name("type").value(attachments.type.toString())
|
||||
name("encryption").apply {
|
||||
attachments.attachmentInfo?.encryption?.let { encryption ->
|
||||
beginObject()
|
||||
name("key").value(encryption.key)
|
||||
name("iv").value(encryption.iv)
|
||||
endObject()
|
||||
} ?: nullValue()
|
||||
}
|
||||
endObject()
|
||||
}
|
||||
endArray()
|
||||
endObject()
|
||||
flush()
|
||||
}
|
||||
}
|
||||
|
||||
fun awaitDownload() {
|
||||
downloadThreadExecutor.shutdown()
|
||||
downloadThreadExecutor.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
|
||||
writeThreadExecutor.shutdown()
|
||||
writeThreadExecutor.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
|
||||
}
|
||||
|
||||
fun close() {
|
||||
if (exportFormat != ExportFormat.TEXT) {
|
||||
jsonDataWriter.endArray()
|
||||
jsonDataWriter.endObject()
|
||||
jsonDataWriter.flush()
|
||||
jsonDataWriter.close()
|
||||
}
|
||||
|
||||
if (exportFormat == ExportFormat.JSON) {
|
||||
conversationJsonDataFile.inputStream().use {
|
||||
it.copyTo(outputFileStream)
|
||||
}
|
||||
}
|
||||
|
||||
if (exportFormat == ExportFormat.HTML) {
|
||||
//write the json file
|
||||
outputFileStream.write("<script type=\"application/json\" class=\"exported_content\">".toByteArray())
|
||||
|
||||
(XposedHelpers.newInstance(
|
||||
Base64OutputStream::class.java,
|
||||
outputFileStream,
|
||||
android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
|
||||
true
|
||||
) as OutputStream).let { outputStream ->
|
||||
val deflateOutputStream = DeflaterOutputStream(outputStream, Deflater(Deflater.BEST_COMPRESSION, true), true)
|
||||
conversationJsonDataFile.inputStream().use {
|
||||
it.copyTo(deflateOutputStream)
|
||||
}
|
||||
deflateOutputStream.finish()
|
||||
outputStream.flush()
|
||||
}
|
||||
|
||||
outputFileStream.write("</script>\n".toByteArray())
|
||||
printLog("writing template...")
|
||||
|
||||
runCatching {
|
||||
ZipFile(context.bridgeClient.getApplicationApkPath()).use { apkFile ->
|
||||
//export rawinflate.js
|
||||
apkFile.getEntry("assets/web/rawinflate.js")?.let { entry ->
|
||||
outputFileStream.write("<script>".toByteArray())
|
||||
apkFile.getInputStream(entry).copyTo(outputFileStream)
|
||||
outputFileStream.write("</script>\n".toByteArray())
|
||||
}
|
||||
|
||||
//export avenir next font
|
||||
apkFile.getEntry("assets/web/avenir_next_medium.ttf")?.let { entry ->
|
||||
val encodedFontData = Base64.Default.encode(apkFile.getInputStream(entry).readBytes())
|
||||
outputFileStream.write("""
|
||||
<style>
|
||||
@font-face {
|
||||
font-family: 'Avenir Next';
|
||||
src: url('data:font/truetype;charset=utf-8;base64, $encodedFontData');
|
||||
font-weight: normal;
|
||||
font-style: normal;
|
||||
}
|
||||
</style>
|
||||
""".trimIndent().toByteArray())
|
||||
}
|
||||
|
||||
apkFile.getEntry("assets/web/export_template.html")?.let { entry ->
|
||||
apkFile.getInputStream(entry).copyTo(outputFileStream)
|
||||
}
|
||||
|
||||
apkFile.close()
|
||||
}
|
||||
}.onFailure {
|
||||
throw Throwable("Failed to read template from apk", it)
|
||||
}
|
||||
|
||||
outputFileStream.write("</html>".toByteArray())
|
||||
}
|
||||
|
||||
outputFileStream.flush()
|
||||
outputFileStream.close()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package me.rhunk.snapenhance.core.messaging;
|
||||
|
||||
enum class ExportFormat(
|
||||
val extension: String,
|
||||
){
|
||||
JSON("json"),
|
||||
TEXT("txt"),
|
||||
HTML("html");
|
||||
}
|
||||
@ -1,337 +0,0 @@
|
||||
package me.rhunk.snapenhance.core.messaging
|
||||
|
||||
import android.os.Environment
|
||||
import android.util.Base64InputStream
|
||||
import android.util.Base64OutputStream
|
||||
import com.google.gson.JsonArray
|
||||
import com.google.gson.JsonNull
|
||||
import com.google.gson.JsonObject
|
||||
import de.robv.android.xposed.XposedHelpers
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import me.rhunk.snapenhance.common.BuildConfig
|
||||
import me.rhunk.snapenhance.common.data.ContentType
|
||||
import me.rhunk.snapenhance.common.data.FileType
|
||||
import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
|
||||
import me.rhunk.snapenhance.common.database.impl.FriendInfo
|
||||
import me.rhunk.snapenhance.common.util.protobuf.ProtoReader
|
||||
import me.rhunk.snapenhance.common.util.snap.MediaDownloaderHelper
|
||||
import me.rhunk.snapenhance.common.util.snap.RemoteMediaResolver
|
||||
import me.rhunk.snapenhance.core.ModContext
|
||||
import me.rhunk.snapenhance.core.features.impl.downloader.decoder.AttachmentType
|
||||
import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder
|
||||
import me.rhunk.snapenhance.core.wrapper.impl.Message
|
||||
import me.rhunk.snapenhance.core.wrapper.impl.SnapUUID
|
||||
import java.io.BufferedInputStream
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Collections
|
||||
import java.util.Date
|
||||
import java.util.Locale
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.zip.Deflater
|
||||
import java.util.zip.DeflaterInputStream
|
||||
import java.util.zip.DeflaterOutputStream
|
||||
import java.util.zip.ZipFile
|
||||
import kotlin.io.encoding.Base64
|
||||
import kotlin.io.encoding.ExperimentalEncodingApi
|
||||
|
||||
|
||||
enum class ExportFormat(
|
||||
val extension: String,
|
||||
){
|
||||
JSON("json"),
|
||||
TEXT("txt"),
|
||||
HTML("html");
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalEncodingApi::class)
|
||||
class MessageExporter(
|
||||
private val context: ModContext,
|
||||
private val outputFile: File,
|
||||
private val friendFeedEntry: FriendFeedEntry,
|
||||
private val mediaToDownload: List<ContentType>? = null,
|
||||
private val printLog: (String) -> Unit = {},
|
||||
) {
|
||||
private lateinit var conversationParticipants: Map<String, FriendInfo>
|
||||
private lateinit var messages: List<Message>
|
||||
|
||||
fun readMessages(messages: List<Message>) {
|
||||
conversationParticipants =
|
||||
context.database.getConversationParticipants(friendFeedEntry.key!!)
|
||||
?.mapNotNull {
|
||||
context.database.getFriendInfo(it)
|
||||
}?.associateBy { it.userId!! } ?: emptyMap()
|
||||
|
||||
if (conversationParticipants.isEmpty())
|
||||
throw Throwable("Failed to get conversation participants for ${friendFeedEntry.key}")
|
||||
|
||||
this.messages = messages.sortedBy { it.orderKey }
|
||||
}
|
||||
|
||||
private fun serializeMessageContent(message: Message): String? {
|
||||
return if (message.messageContent!!.contentType == ContentType.CHAT) {
|
||||
ProtoReader(message.messageContent!!.content!!).getString(2, 1) ?: "Failed to parse message"
|
||||
} else null
|
||||
}
|
||||
|
||||
private fun exportText(output: OutputStream) {
|
||||
val writer = output.bufferedWriter()
|
||||
writer.write("Conversation key: ${friendFeedEntry.key}\n")
|
||||
writer.write("Conversation Name: ${friendFeedEntry.feedDisplayName}\n")
|
||||
writer.write("Participants:\n")
|
||||
conversationParticipants.forEach { (userId, friendInfo) ->
|
||||
writer.write(" $userId: ${friendInfo.displayName}\n")
|
||||
}
|
||||
|
||||
writer.write("\nMessages:\n")
|
||||
messages.forEach { message ->
|
||||
val sender = conversationParticipants[message.senderId.toString()]
|
||||
val senderUsername = sender?.usernameForSorting ?: message.senderId.toString()
|
||||
val senderDisplayName = sender?.displayName ?: message.senderId.toString()
|
||||
val messageContent = serializeMessageContent(message) ?: message.messageContent!!.contentType?.name
|
||||
val date = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH).format(Date(message.messageMetadata!!.createdAt!!))
|
||||
writer.write("[$date] - $senderDisplayName (${senderUsername}): $messageContent\n")
|
||||
}
|
||||
writer.flush()
|
||||
}
|
||||
|
||||
private suspend fun exportHtml(output: OutputStream) {
|
||||
val downloadMediaCacheFolder = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "SnapEnhance/cache").also { it.mkdirs() }
|
||||
val mediaFiles = Collections.synchronizedMap(mutableMapOf<String, Pair<FileType, File>>())
|
||||
val threadPool = Executors.newFixedThreadPool(15)
|
||||
|
||||
withContext(Dispatchers.IO) {
|
||||
var processCount = 0
|
||||
|
||||
fun updateProgress(type: String) {
|
||||
val total = messages.filter {
|
||||
mediaToDownload?.contains(it.messageContent!!.contentType) ?: false
|
||||
}.size
|
||||
processCount++
|
||||
printLog("$type $processCount/$total")
|
||||
}
|
||||
|
||||
messages.filter {
|
||||
mediaToDownload?.contains(it.messageContent!!.contentType) ?: false
|
||||
}.forEach { message ->
|
||||
threadPool.execute {
|
||||
MessageDecoder.decode(message.messageContent!!).forEach decode@{ attachment ->
|
||||
val protoMediaReference = Base64.UrlSafe.decode(attachment.mediaUrlKey ?: return@decode)
|
||||
|
||||
runCatching {
|
||||
RemoteMediaResolver.downloadBoltMedia(protoMediaReference, decryptionCallback = {
|
||||
(attachment.attachmentInfo?.encryption?.decryptInputStream(it) ?: it)
|
||||
}) { downloadedInputStream, _ ->
|
||||
downloadedInputStream.use { inputStream ->
|
||||
MediaDownloaderHelper.getSplitElements(inputStream) { type, splitInputStream ->
|
||||
val fileName = "${type}_${Base64.UrlSafe.encode(protoMediaReference).replace("=", "")}"
|
||||
val bufferedInputStream = BufferedInputStream(splitInputStream)
|
||||
val fileType = MediaDownloaderHelper.getFileType(bufferedInputStream)
|
||||
val mediaFile = File(downloadMediaCacheFolder, "$fileName.${fileType.fileExtension}")
|
||||
|
||||
FileOutputStream(mediaFile).use { fos ->
|
||||
bufferedInputStream.copyTo(fos)
|
||||
}
|
||||
|
||||
mediaFiles[fileName] = fileType to mediaFile
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
updateProgress("downloaded")
|
||||
}.onFailure {
|
||||
printLog("failed to download media for ${message.messageDescriptor!!.conversationId}_${message.orderKey}")
|
||||
context.log.error("failed to download media for ${message.messageDescriptor!!.conversationId}_${message.orderKey}", it)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
threadPool.shutdown()
|
||||
threadPool.awaitTermination(30, TimeUnit.DAYS)
|
||||
processCount = 0
|
||||
|
||||
printLog("writing downloaded medias...")
|
||||
|
||||
//write the head of the html file
|
||||
output.write("""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title></title>
|
||||
</head>
|
||||
""".trimIndent().toByteArray())
|
||||
|
||||
output.write("<!-- This file was generated by SnapEnhance ${BuildConfig.VERSION_NAME} -->\n".toByteArray())
|
||||
|
||||
mediaFiles.forEach { (key, filePair) ->
|
||||
output.write("<div class=\"media-$key\"><!-- ".toByteArray())
|
||||
filePair.second.inputStream().use { inputStream ->
|
||||
val deflateInputStream = DeflaterInputStream(inputStream, Deflater(Deflater.BEST_COMPRESSION, true))
|
||||
(XposedHelpers.newInstance(
|
||||
Base64InputStream::class.java,
|
||||
deflateInputStream,
|
||||
android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
|
||||
true
|
||||
) as InputStream).copyTo(output)
|
||||
}
|
||||
output.write(" --></div>\n".toByteArray())
|
||||
output.flush()
|
||||
updateProgress("wrote")
|
||||
}
|
||||
printLog("writing json conversation data...")
|
||||
|
||||
//write the json file
|
||||
output.write("<script type=\"application/json\" class=\"exported_content\">".toByteArray())
|
||||
|
||||
val b64os = (XposedHelpers.newInstance(
|
||||
Base64OutputStream::class.java,
|
||||
output,
|
||||
android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
|
||||
true
|
||||
) as OutputStream)
|
||||
val deflateOutputStream = DeflaterOutputStream(b64os, Deflater(Deflater.BEST_COMPRESSION, true), true)
|
||||
exportJson(deflateOutputStream)
|
||||
deflateOutputStream.finish()
|
||||
b64os.flush()
|
||||
|
||||
output.write("</script>\n".toByteArray())
|
||||
|
||||
printLog("writing template...")
|
||||
|
||||
runCatching {
|
||||
ZipFile(context.bridgeClient.getApplicationApkPath()).use { apkFile ->
|
||||
//export rawinflate.js
|
||||
apkFile.getEntry("assets/web/rawinflate.js")?.let { entry ->
|
||||
output.write("<script>".toByteArray())
|
||||
apkFile.getInputStream(entry).copyTo(output)
|
||||
output.write("</script>\n".toByteArray())
|
||||
}
|
||||
|
||||
//export avenir next font
|
||||
apkFile.getEntry("assets/web/avenir_next_medium.ttf")?.let { entry ->
|
||||
val encodedFontData = Base64.Default.encode(apkFile.getInputStream(entry).readBytes())
|
||||
output.write("""
|
||||
<style>
|
||||
@font-face {
|
||||
font-family: 'Avenir Next';
|
||||
src: url('data:font/truetype;charset=utf-8;base64, $encodedFontData');
|
||||
font-weight: normal;
|
||||
font-style: normal;
|
||||
}
|
||||
</style>
|
||||
""".trimIndent().toByteArray())
|
||||
}
|
||||
|
||||
apkFile.getEntry("assets/web/export_template.html")?.let { entry ->
|
||||
apkFile.getInputStream(entry).copyTo(output)
|
||||
}
|
||||
|
||||
apkFile.close()
|
||||
}
|
||||
}.onFailure {
|
||||
throw Throwable("Failed to read template from apk", it)
|
||||
}
|
||||
|
||||
output.write("</html>".toByteArray())
|
||||
output.close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun exportJson(output: OutputStream) {
|
||||
val rootObject = JsonObject().apply {
|
||||
addProperty("conversationId", friendFeedEntry.key)
|
||||
addProperty("conversationName", friendFeedEntry.feedDisplayName)
|
||||
|
||||
var index = 0
|
||||
val participants = mutableMapOf<String, Int>()
|
||||
|
||||
add("participants", JsonObject().apply {
|
||||
conversationParticipants.forEach { (userId, friendInfo) ->
|
||||
add(userId, JsonObject().apply {
|
||||
addProperty("id", index)
|
||||
addProperty("displayName", friendInfo.displayName)
|
||||
addProperty("username", friendInfo.usernameForSorting)
|
||||
addProperty("bitmojiSelfieId", friendInfo.bitmojiSelfieId)
|
||||
})
|
||||
participants[userId] = index++
|
||||
}
|
||||
})
|
||||
add("messages", JsonArray().apply {
|
||||
messages.forEach { message ->
|
||||
add(JsonObject().apply {
|
||||
addProperty("orderKey", message.orderKey)
|
||||
addProperty("senderId", participants.getOrDefault(message.senderId.toString(), -1))
|
||||
addProperty("type", message.messageContent!!.contentType.toString())
|
||||
|
||||
fun addUUIDList(name: String, list: List<SnapUUID>) {
|
||||
add(name, JsonArray().apply {
|
||||
list.map { participants.getOrDefault(it.toString(), -1) }.forEach { add(it) }
|
||||
})
|
||||
}
|
||||
|
||||
addUUIDList("savedBy", message.messageMetadata!!.savedBy!!)
|
||||
addUUIDList("seenBy", message.messageMetadata!!.seenBy!!)
|
||||
addUUIDList("openedBy", message.messageMetadata!!.openedBy!!)
|
||||
|
||||
add("reactions", JsonObject().apply {
|
||||
message.messageMetadata!!.reactions!!.forEach { reaction ->
|
||||
addProperty(
|
||||
participants.getOrDefault(reaction.userId.toString(), -1L).toString(),
|
||||
reaction.reactionId
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
addProperty("createdTimestamp", message.messageMetadata!!.createdAt)
|
||||
addProperty("readTimestamp", message.messageMetadata!!.readAt)
|
||||
addProperty("serializedContent", serializeMessageContent(message))
|
||||
addProperty("rawContent", Base64.UrlSafe.encode(message.messageContent!!.content!!))
|
||||
|
||||
add("attachments", JsonArray().apply {
|
||||
MessageDecoder.decode(message.messageContent!!)
|
||||
.forEach attachments@{ attachments ->
|
||||
if (attachments.type == AttachmentType.STICKER) //TODO: implement stickers
|
||||
return@attachments
|
||||
add(JsonObject().apply {
|
||||
addProperty("key", attachments.mediaUrlKey?.replace("=", ""))
|
||||
addProperty("type", attachments.type.toString())
|
||||
add("encryption", attachments.attachmentInfo?.encryption?.let { encryption ->
|
||||
JsonObject().apply {
|
||||
addProperty("key", encryption.key)
|
||||
addProperty("iv", encryption.iv)
|
||||
}
|
||||
} ?: JsonNull.INSTANCE)
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
output.write(context.gson.toJson(rootObject).toByteArray())
|
||||
output.flush()
|
||||
}
|
||||
|
||||
suspend fun exportTo(exportFormat: ExportFormat) {
|
||||
withContext(Dispatchers.IO) {
|
||||
FileOutputStream(outputFile).apply {
|
||||
when (exportFormat) {
|
||||
ExportFormat.HTML -> exportHtml(this)
|
||||
ExportFormat.JSON -> exportJson(this)
|
||||
ExportFormat.TEXT -> exportText(this)
|
||||
}
|
||||
close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -3,6 +3,7 @@ package me.rhunk.snapenhance.core.wrapper.impl
|
||||
import me.rhunk.snapenhance.common.data.MessageUpdate
|
||||
import me.rhunk.snapenhance.core.ModContext
|
||||
import me.rhunk.snapenhance.core.util.CallbackBuilder
|
||||
import me.rhunk.snapenhance.core.util.ktx.getObjectField
|
||||
import me.rhunk.snapenhance.core.util.ktx.setObjectField
|
||||
import me.rhunk.snapenhance.core.wrapper.AbstractWrapper
|
||||
|
||||
@ -18,6 +19,7 @@ class ConversationManager(
|
||||
private val fetchConversationWithMessagesPaginatedMethod by lazy { findMethodByName("fetchConversationWithMessagesPaginated") }
|
||||
private val fetchConversationWithMessagesMethod by lazy { findMethodByName("fetchConversationWithMessages") }
|
||||
private val fetchMessageByServerId by lazy { findMethodByName("fetchMessageByServerId") }
|
||||
private val fetchMessagesByServerIds by lazy { findMethodByName("fetchMessagesByServerIds") }
|
||||
private val displayedMessagesMethod by lazy { findMethodByName("displayedMessages") }
|
||||
private val fetchMessage by lazy { findMethodByName("fetchMessage") }
|
||||
|
||||
@ -105,4 +107,25 @@ class ConversationManager(
|
||||
}.build()
|
||||
)
|
||||
}
|
||||
|
||||
fun fetchMessagesByServerIds(conversationId: String, serverMessageIds: List<Long>, onSuccess: (List<Message>) -> Unit, onError: (error: String) -> Unit) {
|
||||
fetchMessagesByServerIds.invoke(
|
||||
instanceNonNull(),
|
||||
serverMessageIds.map {
|
||||
CallbackBuilder.createEmptyObject(context.classCache.serverMessageIdentifier.constructors.first())?.apply {
|
||||
setObjectField("mServerConversationId", conversationId.toSnapUUID().instanceNonNull())
|
||||
setObjectField("mServerMessageId", it)
|
||||
}
|
||||
},
|
||||
CallbackBuilder(context.mappings.getMappedClass("callbacks", "FetchMessagesByServerIdsCallback"))
|
||||
.override("onSuccess") { param ->
|
||||
onSuccess(param.arg<List<*>>(0).mapNotNull {
|
||||
Message(it?.getObjectField("mMessage") ?: return@mapNotNull null)
|
||||
})
|
||||
}
|
||||
.override("onError") {
|
||||
onError(it.arg<Any>(0).toString())
|
||||
}.build()
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
package me.rhunk.snapenhance.core.wrapper.impl
|
||||
|
||||
import me.rhunk.snapenhance.common.data.ContentType
|
||||
import me.rhunk.snapenhance.common.data.MessageState
|
||||
import me.rhunk.snapenhance.common.util.protobuf.ProtoReader
|
||||
import me.rhunk.snapenhance.core.wrapper.AbstractWrapper
|
||||
import org.mozilla.javascript.annotations.JSGetter
|
||||
import org.mozilla.javascript.annotations.JSSetter
|
||||
@ -18,4 +20,8 @@ class Message(obj: Any?) : AbstractWrapper(obj) {
|
||||
var messageMetadata by field("mMetadata") { MessageMetadata(it) }
|
||||
@get:JSGetter @set:JSSetter
|
||||
var messageState by enum("mState", MessageState.COMMITTED)
|
||||
|
||||
fun serialize() = if (messageContent!!.contentType == ContentType.CHAT) {
|
||||
ProtoReader(messageContent!!.content!!).getString(2, 1) ?: "Failed to parse message"
|
||||
} else null
|
||||
}
|
||||
Reference in New Issue
Block a user