refactor: message logger

- send timestamp
- sender/conversation usernames
This commit is contained in:
rhunk 2024-06-02 10:12:46 +02:00
parent cb51da8166
commit 6d3e5ed79c
5 changed files with 181 additions and 102 deletions

View File

@ -20,6 +20,7 @@ import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.text.font.FontStyle
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.navigation.NavBackStackEntry
@ -28,6 +29,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import me.rhunk.snapenhance.bridge.DownloadCallback
import me.rhunk.snapenhance.common.bridge.wrapper.ConversationInfo
import me.rhunk.snapenhance.common.bridge.wrapper.LoggedMessage
import me.rhunk.snapenhance.common.bridge.wrapper.LoggerWrapper
import me.rhunk.snapenhance.common.data.ContentType
@ -43,12 +45,8 @@ import me.rhunk.snapenhance.core.features.impl.downloader.decoder.DecodedAttachm
import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder
import me.rhunk.snapenhance.download.DownloadProcessor
import me.rhunk.snapenhance.storage.findFriend
import me.rhunk.snapenhance.storage.getFriendInfo
import me.rhunk.snapenhance.storage.getGroupInfo
import me.rhunk.snapenhance.ui.manager.Routes
import java.nio.ByteBuffer
import java.text.DateFormat
import java.util.UUID
import kotlin.math.absoluteValue
@ -58,15 +56,12 @@ class LoggerHistoryRoot : Routes.Route() {
private var stringFilter by mutableStateOf("")
private var reverseOrder by mutableStateOf(true)
private inline fun decodeMessage(message: LoggedMessage, result: (senderId: String?, contentType: ContentType, messageReader: ProtoReader, attachments: List<DecodedAttachment>) -> Unit) {
private inline fun decodeMessage(message: LoggedMessage, result: (contentType: ContentType, messageReader: ProtoReader, attachments: List<DecodedAttachment>) -> Unit) {
runCatching {
val messageObject = JsonParser.parseString(String(message.messageData, Charsets.UTF_8)).asJsonObject
val senderId = messageObject.getAsJsonObject("mSenderId")?.getAsJsonArray("mId")?.map { it.asByte }?.toByteArray()?.let {
ByteBuffer.wrap(it).run { UUID(long, long) }.toString()
}
val messageContent = messageObject.getAsJsonObject("mMessageContent")
val messageReader = messageContent.getAsJsonArray("mContent").map { it.asByte }.toByteArray().let { ProtoReader(it) }
result(senderId, ContentType.fromMessageContainer(messageReader) ?: ContentType.UNKNOWN, messageReader, MessageDecoder.decode(messageContent))
result(ContentType.fromMessageContainer(messageReader) ?: ContentType.UNKNOWN, messageReader, MessageDecoder.decode(messageContent))
}.onFailure {
context.log.error("Failed to decode message", it)
}
@ -129,12 +124,10 @@ class LoggerHistoryRoot : Routes.Route() {
LaunchedEffect(Unit, message) {
runCatching {
decodeMessage(message) { senderId, contentType, messageReader, attachments ->
val senderUsername = senderId?.let { context.database.getFriendInfo(it)?.mutableUsername } ?: translation["unknown_sender"]
decodeMessage(message) { contentType, messageReader, attachments ->
@Composable
fun ContentHeader() {
Text("$senderUsername (${contentType.toString().lowercase()})", modifier = Modifier.padding(end = 4.dp), fontWeight = FontWeight.ExtraLight)
Text("${message.username} (${contentType.toString().lowercase()}) - ${DateFormat.getDateTimeInstance().format(message.sendTimestamp)}", modifier = Modifier.padding(end = 4.dp), fontWeight = FontWeight.ExtraLight)
}
if (contentType == ContentType.CHAT) {
@ -187,7 +180,7 @@ class LoggerHistoryRoot : Routes.Route() {
ElevatedButton(onClick = {
context.coroutineScope.launch {
runCatching {
downloadAttachment(message.timestamp, attachment)
downloadAttachment(message.sendTimestamp, attachment)
}.onFailure {
context.log.error("Failed to download attachment", it)
context.shortToast(translation["download_attachment_failed_toast"])
@ -232,17 +225,26 @@ class LoggerHistoryRoot : Routes.Route() {
expanded = expanded,
onExpandedChange = { expanded = it },
) {
fun formatConversationId(conversationId: String?): String? {
if (conversationId == null) return null
return context.database.getGroupInfo(conversationId)?.name?.let {
fun formatConversationInfo(conversationInfo: ConversationInfo?): String? {
if (conversationInfo == null) return null
return conversationInfo.groupTitle?.let {
translation.format("list_group_format", "name" to it)
} ?: context.database.findFriend(conversationId)?.let {
translation.format("list_friend_format", "name" to (it.displayName?.let { name -> "$name (${it.mutableUsername})" } ?: it.mutableUsername))
} ?: conversationId
} ?: conversationInfo.usernames.takeIf { it.size > 1 }?.let {
translation.format("list_friend_format", "name" to ("(" + it.joinToString(", ") + ")"))
} ?: context.database.findFriend(conversationInfo.conversationId)?.let {
translation.format("list_friend_format", "name" to "(" + (conversationInfo.usernames + listOf(it.mutableUsername)).toSet().joinToString(", ") + ")")
} ?: conversationInfo.usernames.firstOrNull()?.let {
translation.format("list_friend_format", "name" to "($it)")
}
}
val selectedConversationInfo by rememberAsyncMutableState(defaultValue = null, keys = arrayOf(selectedConversation)) {
selectedConversation?.let { loggerWrapper.getConversationInfo(it) }
}
OutlinedTextField(
value = remember(selectedConversation) { formatConversationId(selectedConversation) ?: "Select a conversation" },
value = remember(selectedConversationInfo) { formatConversationInfo(selectedConversationInfo) ?: "Select a conversation" },
onValueChange = {},
readOnly = true,
modifier = Modifier
@ -260,7 +262,15 @@ class LoggerHistoryRoot : Routes.Route() {
selectedConversation = conversationId
expanded = false
}, text = {
Text(remember(conversationId) { formatConversationId(conversationId) ?: "Unknown conversation" })
val conversationInfo by rememberAsyncMutableState(defaultValue = null, keys = arrayOf(conversationId)) {
formatConversationInfo(loggerWrapper.getConversationInfo(conversationId))
}
Text(
text = remember(conversationInfo) { conversationInfo ?: conversationId },
fontWeight = if (conversationId == selectedConversation) FontWeight.Bold else FontWeight.Normal,
overflow = TextOverflow.Ellipsis
)
})
}
}
@ -320,7 +330,7 @@ class LoggerHistoryRoot : Routes.Route() {
) { messageData ->
if (stringFilter.isEmpty()) return@fetchMessages true
var isMatch = false
decodeMessage(messageData) { _, contentType, messageReader, _ ->
decodeMessage(messageData) { contentType, messageReader, _ ->
if (contentType == ContentType.CHAT) {
val content = messageReader.getString(2, 1) ?: return@decodeMessage
isMatch = content.contains(stringFilter, ignoreCase = true)
@ -332,7 +342,7 @@ class LoggerHistoryRoot : Routes.Route() {
hasReachedEnd = true
return@withContext
}
lastFetchMessageTimestamp = newMessages.lastOrNull()?.timestamp ?: return@withContext
lastFetchMessageTimestamp = newMessages.lastOrNull()?.sendTimestamp ?: return@withContext
withContext(Dispatchers.Main) {
messages.addAll(newMessages)
}

View File

@ -0,0 +1,11 @@
package me.rhunk.snapenhance.bridge.logger;
parcelable BridgeLoggedMessage {
long messageId;
String conversationId;
String userId;
String username;
long sendTimestamp;
@nullable String groupTitle;
byte[] messageData;
}

View File

@ -1,5 +1,7 @@
package me.rhunk.snapenhance.bridge.logger;
import me.rhunk.snapenhance.bridge.logger.BridgeLoggedMessage;
interface LoggerInterface {
/**
* Get the ids of the messages that are logged
@ -15,7 +17,7 @@ interface LoggerInterface {
/**
* Add a message to the message logger database if it is not already there
*/
oneway void addMessage(String conversationId, long id, in byte[] message);
oneway void addMessage(in BridgeLoggedMessage message);
/**
* Delete a message from the message logger database

View File

@ -6,6 +6,7 @@ import android.database.sqlite.SQLiteDatabase
import com.google.gson.GsonBuilder
import com.google.gson.JsonObject
import kotlinx.coroutines.*
import me.rhunk.snapenhance.bridge.logger.BridgeLoggedMessage
import me.rhunk.snapenhance.bridge.logger.LoggerInterface
import me.rhunk.snapenhance.common.bridge.InternalFileHandleType
import me.rhunk.snapenhance.common.data.StoryData
@ -26,10 +27,22 @@ class LoggedMessageEdit(
class LoggedMessage(
val messageId: Long,
val timestamp: Long,
val conversationId: String,
val userId: String,
val username: String,
val sendTimestamp: Long,
val addedTimestamp: Long,
val groupTitle: String?,
val messageData: ByteArray,
)
class ConversationInfo(
val conversationId: String,
val participantSize: Int,
val groupTitle: String?,
val usernames: List<String>
)
class TrackerLog(
val id: Int,
val timestamp: Long,
@ -77,9 +90,13 @@ class LoggerWrapper(
SQLiteDatabaseHelper.createTablesFromSchema(openedDatabase, mapOf(
"messages" to listOf(
"id INTEGER PRIMARY KEY",
"added_timestamp BIGINT",
"conversation_id VARCHAR",
"message_id BIGINT",
"conversation_id VARCHAR",
"user_id CHAR(36)",
"username VARCHAR",
"send_timestamp BIGINT",
"added_timestamp BIGINT",
"group_title VARCHAR",
"message_data BLOB"
),
"chat_edits" to listOf(
@ -150,67 +167,67 @@ class LoggerWrapper(
}
}
override fun addMessage(conversationId: String, messageId: Long, serializedMessage: ByteArray) {
val hasMessage = database.rawQuery("SELECT message_id FROM messages WHERE conversation_id = ? AND message_id = ?", arrayOf(conversationId, messageId.toString())).use {
override fun addMessage(bridgeLoggedMessage: BridgeLoggedMessage) {
val hasMessage = database.rawQuery("SELECT message_id FROM messages WHERE conversation_id = ? AND message_id = ?", arrayOf(bridgeLoggedMessage.conversationId, bridgeLoggedMessage.messageId.toString())).use {
it.moveToFirst()
it.count > 0
}
if (!hasMessage) {
runBlocking {
withContext(coroutineScope.coroutineContext) {
database.insert("messages", null, ContentValues().apply {
put("added_timestamp", System.currentTimeMillis())
put("conversation_id", conversationId)
put("message_id", messageId)
put("message_data", serializedMessage)
})
}
runBlocking(coroutineScope.coroutineContext) {
database.insert("messages", null, ContentValues().apply {
put("message_id", bridgeLoggedMessage.messageId)
put("conversation_id", bridgeLoggedMessage.conversationId)
put("user_id", bridgeLoggedMessage.userId)
put("username", bridgeLoggedMessage.username)
put("send_timestamp", bridgeLoggedMessage.sendTimestamp)
put("added_timestamp", System.currentTimeMillis())
put("group_title", bridgeLoggedMessage.groupTitle)
put("message_data", bridgeLoggedMessage.messageData)
})
}
}
// handle message edits
runBlocking {
withContext(coroutineScope.coroutineContext) {
runCatching {
val messageObject = gson.fromJson(
serializedMessage.toString(Charsets.UTF_8),
JsonObject::class.java
)
if (messageObject.getAsJsonObject("mMessageContent")
?.getAsJsonPrimitive("mContentType")?.asString != "CHAT"
) return@withContext
runBlocking(coroutineScope.coroutineContext) {
runCatching {
val messageObject = gson.fromJson(
bridgeLoggedMessage.messageData.toString(Charsets.UTF_8),
JsonObject::class.java
)
if (messageObject.getAsJsonObject("mMessageContent")
?.getAsJsonPrimitive("mContentType")?.asString != "CHAT"
) return@runBlocking
val metadata = messageObject.getAsJsonObject("mMetadata")
if (metadata.get("mIsEdited")?.asBoolean != true) return@withContext
val metadata = messageObject.getAsJsonObject("mMetadata")
if (metadata.get("mIsEdited")?.asBoolean != true) return@runBlocking
val messageTextContent =
messageObject.getAsJsonObject("mMessageContent")?.getAsJsonArray("mContent")
?.map { it.asByte }?.toByteArray()?.let {
ProtoReader(it).getString(2, 1)
} ?: return@withContext
val messageTextContent =
messageObject.getAsJsonObject("mMessageContent")?.getAsJsonArray("mContent")
?.map { it.asByte }?.toByteArray()?.let {
ProtoReader(it).getString(2, 1)
} ?: return@runBlocking
database.rawQuery(
"SELECT MAX(edit_number), message_text FROM chat_edits WHERE conversation_id = ? AND message_id = ?",
arrayOf(conversationId, messageId.toString())
).use {
it.moveToFirst()
val editNumber = it.getInt(0)
val lastEditedMessage = it.getString(1)
database.rawQuery(
"SELECT MAX(edit_number), message_text FROM chat_edits WHERE conversation_id = ? AND message_id = ?",
arrayOf(bridgeLoggedMessage.conversationId, bridgeLoggedMessage.messageId.toString())
).use {
it.moveToFirst()
val editNumber = it.getInt(0)
val lastEditedMessage = it.getString(1)
if (lastEditedMessage == messageTextContent) return@withContext
if (lastEditedMessage == messageTextContent) return@runBlocking
database.insert("chat_edits", null, ContentValues().apply {
put("edit_number", editNumber + 1)
put("added_timestamp", System.currentTimeMillis())
put("conversation_id", conversationId)
put("message_id", messageId)
put("message_text", messageTextContent)
})
}
}.onFailure {
AbstractLogger.directDebug("Failed to handle message edit: ${it.message}")
database.insert("chat_edits", null, ContentValues().apply {
put("edit_number", editNumber + 1)
put("added_timestamp", System.currentTimeMillis())
put("conversation_id", bridgeLoggedMessage.conversationId)
put("message_id", bridgeLoggedMessage.messageId)
put("message_text", messageTextContent)
})
}
}.onFailure {
AbstractLogger.directDebug("Failed to handle message edit: ${it.message}")
}
}
}
@ -257,18 +274,16 @@ class LoggerWrapper(
}) {
return false
}
runBlocking {
withContext(coroutineScope.coroutineContext) {
database.insert("stories", null, ContentValues().apply {
put("user_id", userId)
put("added_timestamp", System.currentTimeMillis())
put("url", url)
put("posted_timestamp", postedAt)
put("created_timestamp", createdAt)
put("encryption_key", key)
put("encryption_iv", iv)
})
}
runBlocking(coroutineScope.coroutineContext) {
database.insert("stories", null, ContentValues().apply {
put("user_id", userId)
put("added_timestamp", System.currentTimeMillis())
put("url", url)
put("posted_timestamp", postedAt)
put("created_timestamp", createdAt)
put("encryption_key", key)
put("encryption_iv", iv)
})
}
return true
}
@ -282,19 +297,17 @@ class LoggerWrapper(
eventType: String,
data: String
) {
runBlocking {
withContext(coroutineScope.coroutineContext) {
database.insert("tracker_events", null, ContentValues().apply {
put("timestamp", System.currentTimeMillis())
put("conversation_id", conversationId)
put("conversation_title", conversationTitle)
put("is_group", isGroup)
put("username", username)
put("user_id", userId)
put("event_type", eventType)
put("data", data)
})
}
runBlocking(coroutineScope.coroutineContext) {
database.insert("tracker_events", null, ContentValues().apply {
put("timestamp", System.currentTimeMillis())
put("conversation_id", conversationId)
put("conversation_title", conversationTitle)
put("is_group", isGroup)
put("username", username)
put("user_id", userId)
put("event_type", eventType)
put("data", data)
})
}
}
@ -389,6 +402,26 @@ class LoggerWrapper(
}
}
fun getConversationInfo(conversationId: String): ConversationInfo? {
val participantSize = database.rawQuery("SELECT COUNT(DISTINCT user_id) FROM messages WHERE conversation_id = ?", arrayOf(conversationId)).use {
if (!it.moveToFirst()) return null
it.getInt(0)
}
val groupTitle = if (participantSize > 2) database.rawQuery("SELECT group_title FROM messages WHERE conversation_id = ? AND group_title IS NOT NULL LIMIT 1", arrayOf(conversationId)).use {
if (!it.moveToFirst()) return@use null
it.getStringOrNull("group_title")
} else null
val usernames = database.rawQuery("SELECT DISTINCT username FROM messages WHERE conversation_id = ?", arrayOf(conversationId)).use {
val usernames = mutableListOf<String>()
while (it.moveToNext()) {
usernames.add(it.getString(0))
}
usernames
}
return ConversationInfo(conversationId, participantSize, groupTitle, usernames)
}
fun getMessageEdits(conversationId: String, messageId: Long): List<LoggedMessageEdit> {
val edits = mutableListOf<LoggedMessageEdit>()
database.rawQuery(
@ -414,13 +447,18 @@ class LoggerWrapper(
): List<LoggedMessage> {
val messages = mutableListOf<LoggedMessage>()
database.rawQuery(
"SELECT * FROM messages WHERE conversation_id = ? AND added_timestamp ${if (reverseOrder) "<" else ">"} ? ORDER BY added_timestamp ${if (reverseOrder) "DESC" else "ASC"}",
"SELECT * FROM messages WHERE conversation_id = ? AND send_timestamp ${if (reverseOrder) "<" else ">"} ? ORDER BY send_timestamp ${if (reverseOrder) "DESC" else "ASC"}",
arrayOf(conversationId, fromTimestamp.toString())
).use {
while (it.moveToNext() && messages.size < limit) {
val message = LoggedMessage(
messageId = it.getLongOrNull("message_id") ?: continue,
timestamp = it.getLongOrNull("added_timestamp") ?: continue,
conversationId = it.getStringOrNull("conversation_id") ?: continue,
userId = it.getStringOrNull("user_id") ?: continue,
username = it.getStringOrNull("username") ?: continue,
sendTimestamp = it.getLongOrNull("send_timestamp") ?: continue,
addedTimestamp = it.getLongOrNull("added_timestamp") ?: continue,
groupTitle = it.getStringOrNull("group_title"),
messageData = it.getBlobOrNull("message_data") ?: continue
)
if (filter != null && !filter(message)) continue

View File

@ -7,6 +7,7 @@ import android.graphics.drawable.shapes.Shape
import android.os.DeadObjectException
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import me.rhunk.snapenhance.bridge.logger.BridgeLoggedMessage
import me.rhunk.snapenhance.common.data.ContentType
import me.rhunk.snapenhance.common.data.MessageState
import me.rhunk.snapenhance.common.data.QuotedMessageContentStatus
@ -40,6 +41,9 @@ class MessageLogger : Feature("MessageLogger",
private val threadPool = Executors.newFixedThreadPool(10)
private val usernameCache = EvictingMap<String, String>(500) // user id -> username
private val groupTitleCache = EvictingMap<String, String?>(500) // conversation id -> group title
private val cachedIdLinks = EvictingMap<Long, Long>(500) // client id -> server id
private val fetchedMessages = mutableListOf<Long>() // list of unique message ids
private val deletedMessageCache = EvictingMap<Long, JsonObject>(200) // unique message id -> message json object
@ -127,7 +131,21 @@ class MessageLogger : Feature("MessageLogger",
threadPool.execute {
try {
loggerInterface.addMessage(conversationId, uniqueMessageIdentifier, context.gson.toJson(messageInstance).toByteArray(Charsets.UTF_8))
loggerInterface.addMessage(
BridgeLoggedMessage().also {
it.messageId = uniqueMessageIdentifier
it.conversationId = conversationId
it.userId = event.message.senderId.toString()
it.username = usernameCache.getOrPut(it.userId) {
context.database.getFriendInfo(it.userId)?.mutableUsername ?: it.userId
}
it.sendTimestamp = event.message.messageMetadata?.createdAt ?: System.currentTimeMillis()
it.groupTitle = groupTitleCache.getOrPut(conversationId) {
context.database.getFeedEntryByConversationId(conversationId)?.feedDisplayName ?: conversationId
}
it.messageData = context.gson.toJson(messageInstance).toByteArray(Charsets.UTF_8)
}
)
} catch (ignored: DeadObjectException) {}
}