commit 04fcc33264a9652b2580077b079eef68fc80005e
parent 8fd72d60dfb4b188e0ac12bcf2d0cb0cba11ecda
Author: rhunk <101876869+rhunk@users.noreply.github.com>
Date: Sat, 25 Nov 2023 16:29:09 +0100
perf(core/message_exporter): async download
- add retries
Diffstat:
7 files changed, 416 insertions(+), 385 deletions(-)
diff --git a/core/src/main/assets/web/export_template.html b/core/src/main/assets/web/export_template.html
@@ -232,15 +232,18 @@
}
function decodeMedia(element) {
- const decodedData = new Uint8Array(
- inflate(
- base64decode(
- element.innerHTML.substring(5, element.innerHTML.length - 4)
+ try {
+ const decodedData = new Uint8Array(
+ inflate(
+ base64decode(
+ element.innerHTML.substring(5, element.innerHTML.length - 4)
+ )
)
)
- )
-
- return URL.createObjectURL(new Blob([decodedData]))
+ return URL.createObjectURL(new Blob([decodedData]))
+ } catch (e) {
+ return null
+ }
}
function makeMain() {
diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/action/impl/ExportChatMessages.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/action/impl/ExportChatMessages.kt
@@ -5,18 +5,14 @@ import android.content.DialogInterface
import android.os.Environment
import android.text.InputType
import android.widget.EditText
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.Job
-import kotlinx.coroutines.joinAll
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.suspendCancellableCoroutine
+import kotlinx.coroutines.*
import me.rhunk.snapenhance.common.data.ContentType
import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
import me.rhunk.snapenhance.core.action.AbstractAction
import me.rhunk.snapenhance.core.features.impl.messaging.Messaging
import me.rhunk.snapenhance.core.logger.CoreLogger
+import me.rhunk.snapenhance.core.messaging.ConversationExporter
import me.rhunk.snapenhance.core.messaging.ExportFormat
-import me.rhunk.snapenhance.core.messaging.MessageExporter
import me.rhunk.snapenhance.core.ui.ViewAppearanceHelper
import me.rhunk.snapenhance.core.wrapper.impl.Message
import java.io.File
@@ -83,6 +79,7 @@ class ExportChatMessages : AbstractAction() {
context.runOnUiThread {
val mediasToDownload = mutableListOf<ContentType>()
val contentTypes = arrayOf(
+ ContentType.CHAT,
ContentType.SNAP,
ContentType.EXTERNAL_MEDIA,
ContentType.NOTE,
@@ -142,25 +139,54 @@ class ExportChatMessages : AbstractAction() {
}
}
- private suspend fun fetchMessagesPaginated(conversationId: String, lastMessageId: Long, amount: Int): List<Message> = suspendCancellableCoroutine { continuation ->
- context.feature(Messaging::class).conversationManager?.fetchConversationWithMessagesPaginated(conversationId,
- lastMessageId,
- amount, onSuccess = { messages ->
- continuation.resumeWith(Result.success(messages))
- }, onError = {
- continuation.resumeWith(Result.success(emptyList()))
- }) ?: continuation.resumeWith(Result.success(emptyList()))
+ private suspend fun fetchMessagesPaginated(conversationId: String, lastMessageId: Long, amount: Int): List<Message> = runBlocking {
+ for (i in 0..5) {
+ val messages: List<Message>? = suspendCancellableCoroutine { continuation ->
+ context.feature(Messaging::class).conversationManager?.fetchConversationWithMessagesPaginated(conversationId,
+ lastMessageId,
+ amount, onSuccess = { messages ->
+ continuation.resumeWith(Result.success(messages))
+ }, onError = {
+ continuation.resumeWith(Result.success(null))
+ }) ?: continuation.resumeWith(Result.success(null))
+ }
+ if (messages != null) return@runBlocking messages
+ logDialog("Retrying in 1 second...")
+ delay(1000)
+ }
+ logDialog("Failed to fetch messages")
+ emptyList()
}
private suspend fun exportFullConversation(friendFeedEntry: FriendFeedEntry) {
//first fetch the first message
val conversationId = friendFeedEntry.key!!
val conversationName = friendFeedEntry.feedDisplayName ?: friendFeedEntry.friendDisplayName!!.split("|").lastOrNull() ?: "unknown"
+ val conversationParticipants = context.database.getConversationParticipants(friendFeedEntry.key!!)
+ ?.mapNotNull {
+ context.database.getFriendInfo(it)
+ }?.associateBy { it.userId!! } ?: emptyMap()
+
+ val publicFolder = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "SnapEnhance")
+ val outputFile = publicFolder.resolve("conversation_${conversationName}_${System.currentTimeMillis()}.${exportType!!.extension}")
logDialog(context.translation.format("chat_export.exporting_message", "conversation" to conversationName))
- val foundMessages = fetchMessagesPaginated(conversationId, Long.MAX_VALUE, amount = 1).toMutableList()
- var lastMessageId = foundMessages.firstOrNull()?.messageDescriptor?.messageId ?: run {
+ val conversationExporter = ConversationExporter(
+ context = context,
+ friendFeedEntry = friendFeedEntry,
+ conversationParticipants = conversationParticipants,
+ exportFormat = exportType!!,
+ messageTypeFilter = mediaToDownload,
+ cacheFolder = publicFolder.resolve("cache"),
+ outputFile = outputFile,
+ ).apply { init(); printLog = {
+ logDialog(it.toString())
+ } }
+
+ var foundMessageCount = 0
+
+ var lastMessageId = fetchMessagesPaginated(conversationId, Long.MAX_VALUE, amount = 1).firstOrNull()?.messageDescriptor?.messageId ?: run {
logDialog(context.translation["chat_export.no_messages_found"])
return
}
@@ -168,40 +194,28 @@ class ExportChatMessages : AbstractAction() {
while (true) {
val fetchedMessages = fetchMessagesPaginated(conversationId, lastMessageId, amount = 500)
if (fetchedMessages.isEmpty()) break
+ foundMessageCount += fetchedMessages.size
- foundMessages.addAll(fetchedMessages)
- if (amountOfMessages != null && foundMessages.size >= amountOfMessages!!) {
- foundMessages.subList(amountOfMessages!!, foundMessages.size).clear()
+ if (amountOfMessages != null && foundMessageCount >= amountOfMessages!!) {
+ fetchedMessages.subList(0, amountOfMessages!! - foundMessageCount).reversed().forEach { message ->
+ conversationExporter.readMessage(message)
+ }
break
}
+ fetchedMessages.reversed().forEach { message ->
+ conversationExporter.readMessage(message)
+ }
+
fetchedMessages.firstOrNull()?.let {
lastMessageId = it.messageDescriptor!!.messageId!!
}
- setStatus("Exporting (${foundMessages.size} / ${foundMessages.firstOrNull()?.orderKey})")
+ setStatus("Exporting (found ${foundMessageCount})")
}
- val outputFile = File(
- Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS),
- "SnapEnhance/conversation_${conversationName}_${System.currentTimeMillis()}.${exportType!!.extension}"
- ).also { it.parentFile?.mkdirs() }
-
+ if (exportType == ExportFormat.HTML) conversationExporter.awaitDownload()
+ conversationExporter.close()
logDialog(context.translation["chat_export.writing_output"])
-
- runCatching {
- MessageExporter(
- context = context,
- friendFeedEntry = friendFeedEntry,
- outputFile = outputFile,
- mediaToDownload = mediaToDownload,
- printLog = ::logDialog
- ).apply { readMessages(foundMessages) }.exportTo(exportType!!)
- }.onFailure {
- logDialog(context.translation.format("chat_export.export_failed","conversation" to it.message.toString()))
- context.log.error("Failed to export conversation $conversationName", it)
- return
- }
-
dialogLogs.clear()
logDialog("\n" + context.translation.format("chat_export.exported_to",
"path" to outputFile.absolutePath.toString()
diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/ConversationExporter.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/ConversationExporter.kt
@@ -0,0 +1,313 @@
+package me.rhunk.snapenhance.core.messaging
+
+import android.util.Base64InputStream
+import android.util.Base64OutputStream
+import com.google.gson.stream.JsonWriter
+import de.robv.android.xposed.XposedHelpers
+import me.rhunk.snapenhance.common.BuildConfig
+import me.rhunk.snapenhance.common.data.ContentType
+import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
+import me.rhunk.snapenhance.common.database.impl.FriendInfo
+import me.rhunk.snapenhance.common.util.snap.MediaDownloaderHelper
+import me.rhunk.snapenhance.common.util.snap.RemoteMediaResolver
+import me.rhunk.snapenhance.core.ModContext
+import me.rhunk.snapenhance.core.features.impl.downloader.decoder.AttachmentType
+import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder
+import me.rhunk.snapenhance.core.wrapper.impl.Message
+import me.rhunk.snapenhance.core.wrapper.impl.SnapUUID
+import java.io.BufferedInputStream
+import java.io.File
+import java.io.InputStream
+import java.io.OutputStream
+import java.text.DateFormat
+import java.util.Date
+import java.util.concurrent.Executors
+import java.util.zip.Deflater
+import java.util.zip.DeflaterInputStream
+import java.util.zip.DeflaterOutputStream
+import java.util.zip.ZipFile
+import kotlin.io.encoding.Base64
+import kotlin.io.encoding.ExperimentalEncodingApi
+
+@OptIn(ExperimentalEncodingApi::class)
+class ConversationExporter(
+ private val context: ModContext,
+ private val friendFeedEntry: FriendFeedEntry,
+ private val conversationParticipants: Map<String, FriendInfo>,
+ private val exportFormat: ExportFormat,
+ private val messageTypeFilter: List<ContentType>? = null,
+ private val cacheFolder: File,
+ private val outputFile: File
+) {
+ lateinit var printLog: (Any?) -> Unit
+
+ private val downloadThreadExecutor = Executors.newFixedThreadPool(4)
+ private val writeThreadExecutor = Executors.newSingleThreadExecutor()
+
+ private val conversationJsonDataFile by lazy { cacheFolder.resolve("messages.json") }
+ private val jsonDataWriter by lazy { JsonWriter(conversationJsonDataFile.writer()) }
+ private val outputFileStream by lazy { outputFile.outputStream() }
+ private val participants = mutableMapOf<String, Int>()
+
+ fun init() {
+ when (exportFormat) {
+ ExportFormat.TEXT -> {
+ outputFileStream.write("Conversation id: ${friendFeedEntry.key}\n".toByteArray())
+ outputFileStream.write("Conversation name: ${friendFeedEntry.feedDisplayName}\n".toByteArray())
+ outputFileStream.write("Participants:\n".toByteArray())
+ conversationParticipants.forEach { (userId, friendInfo) ->
+ outputFileStream.write(" $userId: ${friendInfo.displayName}\n".toByteArray())
+ }
+ outputFileStream.write("\n\n".toByteArray())
+ }
+ else -> {
+ jsonDataWriter.isHtmlSafe = true
+ jsonDataWriter.serializeNulls = true
+
+ jsonDataWriter.beginObject()
+ jsonDataWriter.name("conversationId").value(friendFeedEntry.key)
+ jsonDataWriter.name("conversationName").value(friendFeedEntry.feedDisplayName)
+
+ var index = 0
+
+ jsonDataWriter.name("participants").apply {
+ beginObject()
+ conversationParticipants.forEach { (userId, friendInfo) ->
+ jsonDataWriter.name(userId).beginObject()
+ jsonDataWriter.name("id").value(index)
+ jsonDataWriter.name("displayName").value(friendInfo.displayName)
+ jsonDataWriter.name("username").value(friendInfo.usernameForSorting)
+ jsonDataWriter.name("bitmojiSelfieId").value(friendInfo.bitmojiSelfieId)
+ jsonDataWriter.endObject()
+ participants[userId] = index++
+ }
+ endObject()
+ }
+
+ jsonDataWriter.name("messages").beginArray()
+
+ if (exportFormat != ExportFormat.HTML) return
+ outputFileStream.write("""
+ <!DOCTYPE html>
+ <html>
+ <head>
+ <meta charset="UTF-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title></title>
+ </head>
+ """.trimIndent().toByteArray())
+
+ outputFileStream.write("<!-- This file was generated by SnapEnhance ${BuildConfig.VERSION_NAME} -->\n</head>".toByteArray())
+
+ outputFileStream.flush()
+ }
+ }
+ }
+
+
+ @OptIn(ExperimentalEncodingApi::class)
+ private fun downloadMedia(message: Message) {
+ downloadThreadExecutor.execute {
+ MessageDecoder.decode(message.messageContent!!).forEach decode@{ attachment ->
+ if (attachment.mediaUrlKey?.isEmpty() == true) return@decode
+ val protoMediaReference = Base64.UrlSafe.decode(attachment.mediaUrlKey ?: return@decode)
+
+ for (i in 0..5) {
+ printLog("downloading ${attachment.mediaUrlKey}... (attempt ${i + 1}/5)")
+ runCatching {
+ RemoteMediaResolver.downloadBoltMedia(protoMediaReference, decryptionCallback = {
+ (attachment.attachmentInfo?.encryption?.decryptInputStream(it) ?: it)
+ }) { downloadedInputStream, _ ->
+ downloadedInputStream.use { inputStream ->
+ MediaDownloaderHelper.getSplitElements(inputStream) { type, splitInputStream ->
+ val mediaKey = "${type}_${Base64.UrlSafe.encode(protoMediaReference).replace("=", "")}"
+ val bufferedInputStream = BufferedInputStream(splitInputStream)
+ val fileType = MediaDownloaderHelper.getFileType(bufferedInputStream)
+ val mediaFile = cacheFolder.resolve("$mediaKey.${fileType.fileExtension}")
+
+ mediaFile.outputStream().use { fos ->
+ bufferedInputStream.copyTo(fos)
+ }
+
+ writeThreadExecutor.execute {
+ outputFileStream.write("<div class=\"media-$mediaKey\"><!-- ".toByteArray())
+ mediaFile.inputStream().use {
+ val deflateInputStream = DeflaterInputStream(it, Deflater(Deflater.BEST_SPEED, true))
+ (XposedHelpers.newInstance(
+ Base64InputStream::class.java,
+ deflateInputStream,
+ android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
+ true
+ ) as InputStream).copyTo(outputFileStream)
+ outputFileStream.write(" --></div>\n".toByteArray())
+ outputFileStream.flush()
+ }
+ }
+ }
+ }
+ }
+ return@decode
+ }.onFailure {
+ printLog("failed to download media ${attachment.mediaUrlKey}. retrying...")
+ it.printStackTrace()
+ }
+ }
+ }
+ }
+ }
+
+ fun readMessage(message: Message) {
+ if (exportFormat == ExportFormat.TEXT) {
+ val (displayName, senderUsername) = conversationParticipants[message.senderId.toString()]?.let {
+ it.displayName to it.mutableUsername
+ } ?: ("" to message.senderId.toString())
+
+ val date = DateFormat.getDateTimeInstance().format(Date(message.messageMetadata!!.createdAt ?: -1))
+ outputFileStream.write("[$date] - $displayName ($senderUsername): ${message.serialize() ?: message.messageContent?.contentType?.name}\n".toByteArray(Charsets.UTF_8))
+ return
+ }
+ val contentType = message.messageContent?.contentType ?: return
+
+ if (messageTypeFilter != null) {
+ if (!messageTypeFilter.contains(contentType)) return
+
+ if (contentType == ContentType.NOTE || contentType == ContentType.SNAP || contentType == ContentType.EXTERNAL_MEDIA) {
+ downloadMedia(message)
+ }
+ }
+
+
+ jsonDataWriter.apply {
+ beginObject()
+ name("orderKey").value(message.orderKey)
+ name("senderId").value(participants.getOrDefault(message.senderId.toString(), -1))
+ name("type").value(message.messageContent!!.contentType.toString())
+
+ fun addUUIDList(name: String, list: List<SnapUUID>) {
+ name(name).beginArray()
+ list.map { participants.getOrDefault(it.toString(), -1) }.forEach { value(it) }
+ endArray()
+ }
+
+ addUUIDList("savedBy", message.messageMetadata!!.savedBy!!)
+ addUUIDList("seenBy", message.messageMetadata!!.seenBy!!)
+ addUUIDList("openedBy", message.messageMetadata!!.openedBy!!)
+
+ name("reactions").beginObject()
+ message.messageMetadata!!.reactions!!.forEach { reaction ->
+ name(participants.getOrDefault(reaction.userId.toString(), -1L).toString()).value(reaction.reactionId)
+ }
+ endObject()
+
+ name("createdTimestamp").value(message.messageMetadata!!.createdAt)
+ name("readTimestamp").value(message.messageMetadata!!.readAt)
+ name("serializedContent").value(message.serialize())
+ name("rawContent").value(Base64.UrlSafe.encode(message.messageContent!!.content!!))
+ name("attachments").beginArray()
+ MessageDecoder.decode(message.messageContent!!)
+ .forEach attachments@{ attachments ->
+ if (attachments.type == AttachmentType.STICKER) //TODO: implement stickers
+ return@attachments
+ beginObject()
+ name("key").value(attachments.mediaUrlKey?.replace("=", ""))
+ name("type").value(attachments.type.toString())
+ name("encryption").apply {
+ attachments.attachmentInfo?.encryption?.let { encryption ->
+ beginObject()
+ name("key").value(encryption.key)
+ name("iv").value(encryption.iv)
+ endObject()
+ } ?: nullValue()
+ }
+ endObject()
+ }
+ endArray()
+ endObject()
+ flush()
+ }
+ }
+
+ fun awaitDownload() {
+ downloadThreadExecutor.shutdown()
+ downloadThreadExecutor.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
+ writeThreadExecutor.shutdown()
+ writeThreadExecutor.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
+ }
+
+ fun close() {
+ if (exportFormat != ExportFormat.TEXT) {
+ jsonDataWriter.endArray()
+ jsonDataWriter.endObject()
+ jsonDataWriter.flush()
+ jsonDataWriter.close()
+ }
+
+ if (exportFormat == ExportFormat.JSON) {
+ conversationJsonDataFile.inputStream().use {
+ it.copyTo(outputFileStream)
+ }
+ }
+
+ if (exportFormat == ExportFormat.HTML) {
+ //write the json file
+ outputFileStream.write("<script type=\"application/json\" class=\"exported_content\">".toByteArray())
+
+ (XposedHelpers.newInstance(
+ Base64OutputStream::class.java,
+ outputFileStream,
+ android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
+ true
+ ) as OutputStream).let { outputStream ->
+ val deflateOutputStream = DeflaterOutputStream(outputStream, Deflater(Deflater.BEST_COMPRESSION, true), true)
+ conversationJsonDataFile.inputStream().use {
+ it.copyTo(deflateOutputStream)
+ }
+ deflateOutputStream.finish()
+ outputStream.flush()
+ }
+
+ outputFileStream.write("</script>\n".toByteArray())
+ printLog("writing template...")
+
+ runCatching {
+ ZipFile(context.bridgeClient.getApplicationApkPath()).use { apkFile ->
+ //export rawinflate.js
+ apkFile.getEntry("assets/web/rawinflate.js")?.let { entry ->
+ outputFileStream.write("<script>".toByteArray())
+ apkFile.getInputStream(entry).copyTo(outputFileStream)
+ outputFileStream.write("</script>\n".toByteArray())
+ }
+
+ //export avenir next font
+ apkFile.getEntry("assets/web/avenir_next_medium.ttf")?.let { entry ->
+ val encodedFontData = Base64.Default.encode(apkFile.getInputStream(entry).readBytes())
+ outputFileStream.write("""
+ <style>
+ @font-face {
+ font-family: 'Avenir Next';
+ src: url('data:font/truetype;charset=utf-8;base64, $encodedFontData');
+ font-weight: normal;
+ font-style: normal;
+ }
+ </style>
+ """.trimIndent().toByteArray())
+ }
+
+ apkFile.getEntry("assets/web/export_template.html")?.let { entry ->
+ apkFile.getInputStream(entry).copyTo(outputFileStream)
+ }
+
+ apkFile.close()
+ }
+ }.onFailure {
+ throw Throwable("Failed to read template from apk", it)
+ }
+
+ outputFileStream.write("</html>".toByteArray())
+ }
+
+ outputFileStream.flush()
+ outputFileStream.close()
+ }
+}+
\ No newline at end of file
diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/ExportFormat.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/ExportFormat.kt
@@ -0,0 +1,9 @@
+package me.rhunk.snapenhance.core.messaging;
+
+enum class ExportFormat(
+ val extension: String,
+){
+ JSON("json"),
+ TEXT("txt"),
+ HTML("html");
+}
diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/MessageExporter.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/messaging/MessageExporter.kt
@@ -1,337 +0,0 @@
-package me.rhunk.snapenhance.core.messaging
-
-import android.os.Environment
-import android.util.Base64InputStream
-import android.util.Base64OutputStream
-import com.google.gson.JsonArray
-import com.google.gson.JsonNull
-import com.google.gson.JsonObject
-import de.robv.android.xposed.XposedHelpers
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.withContext
-import me.rhunk.snapenhance.common.BuildConfig
-import me.rhunk.snapenhance.common.data.ContentType
-import me.rhunk.snapenhance.common.data.FileType
-import me.rhunk.snapenhance.common.database.impl.FriendFeedEntry
-import me.rhunk.snapenhance.common.database.impl.FriendInfo
-import me.rhunk.snapenhance.common.util.protobuf.ProtoReader
-import me.rhunk.snapenhance.common.util.snap.MediaDownloaderHelper
-import me.rhunk.snapenhance.common.util.snap.RemoteMediaResolver
-import me.rhunk.snapenhance.core.ModContext
-import me.rhunk.snapenhance.core.features.impl.downloader.decoder.AttachmentType
-import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder
-import me.rhunk.snapenhance.core.wrapper.impl.Message
-import me.rhunk.snapenhance.core.wrapper.impl.SnapUUID
-import java.io.BufferedInputStream
-import java.io.File
-import java.io.FileOutputStream
-import java.io.InputStream
-import java.io.OutputStream
-import java.text.SimpleDateFormat
-import java.util.Collections
-import java.util.Date
-import java.util.Locale
-import java.util.concurrent.Executors
-import java.util.concurrent.TimeUnit
-import java.util.zip.Deflater
-import java.util.zip.DeflaterInputStream
-import java.util.zip.DeflaterOutputStream
-import java.util.zip.ZipFile
-import kotlin.io.encoding.Base64
-import kotlin.io.encoding.ExperimentalEncodingApi
-
-
-enum class ExportFormat(
- val extension: String,
-){
- JSON("json"),
- TEXT("txt"),
- HTML("html");
-}
-
-@OptIn(ExperimentalEncodingApi::class)
-class MessageExporter(
- private val context: ModContext,
- private val outputFile: File,
- private val friendFeedEntry: FriendFeedEntry,
- private val mediaToDownload: List<ContentType>? = null,
- private val printLog: (String) -> Unit = {},
-) {
- private lateinit var conversationParticipants: Map<String, FriendInfo>
- private lateinit var messages: List<Message>
-
- fun readMessages(messages: List<Message>) {
- conversationParticipants =
- context.database.getConversationParticipants(friendFeedEntry.key!!)
- ?.mapNotNull {
- context.database.getFriendInfo(it)
- }?.associateBy { it.userId!! } ?: emptyMap()
-
- if (conversationParticipants.isEmpty())
- throw Throwable("Failed to get conversation participants for ${friendFeedEntry.key}")
-
- this.messages = messages.sortedBy { it.orderKey }
- }
-
- private fun serializeMessageContent(message: Message): String? {
- return if (message.messageContent!!.contentType == ContentType.CHAT) {
- ProtoReader(message.messageContent!!.content!!).getString(2, 1) ?: "Failed to parse message"
- } else null
- }
-
- private fun exportText(output: OutputStream) {
- val writer = output.bufferedWriter()
- writer.write("Conversation key: ${friendFeedEntry.key}\n")
- writer.write("Conversation Name: ${friendFeedEntry.feedDisplayName}\n")
- writer.write("Participants:\n")
- conversationParticipants.forEach { (userId, friendInfo) ->
- writer.write(" $userId: ${friendInfo.displayName}\n")
- }
-
- writer.write("\nMessages:\n")
- messages.forEach { message ->
- val sender = conversationParticipants[message.senderId.toString()]
- val senderUsername = sender?.usernameForSorting ?: message.senderId.toString()
- val senderDisplayName = sender?.displayName ?: message.senderId.toString()
- val messageContent = serializeMessageContent(message) ?: message.messageContent!!.contentType?.name
- val date = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH).format(Date(message.messageMetadata!!.createdAt!!))
- writer.write("[$date] - $senderDisplayName (${senderUsername}): $messageContent\n")
- }
- writer.flush()
- }
-
- private suspend fun exportHtml(output: OutputStream) {
- val downloadMediaCacheFolder = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "SnapEnhance/cache").also { it.mkdirs() }
- val mediaFiles = Collections.synchronizedMap(mutableMapOf<String, Pair<FileType, File>>())
- val threadPool = Executors.newFixedThreadPool(15)
-
- withContext(Dispatchers.IO) {
- var processCount = 0
-
- fun updateProgress(type: String) {
- val total = messages.filter {
- mediaToDownload?.contains(it.messageContent!!.contentType) ?: false
- }.size
- processCount++
- printLog("$type $processCount/$total")
- }
-
- messages.filter {
- mediaToDownload?.contains(it.messageContent!!.contentType) ?: false
- }.forEach { message ->
- threadPool.execute {
- MessageDecoder.decode(message.messageContent!!).forEach decode@{ attachment ->
- val protoMediaReference = Base64.UrlSafe.decode(attachment.mediaUrlKey ?: return@decode)
-
- runCatching {
- RemoteMediaResolver.downloadBoltMedia(protoMediaReference, decryptionCallback = {
- (attachment.attachmentInfo?.encryption?.decryptInputStream(it) ?: it)
- }) { downloadedInputStream, _ ->
- downloadedInputStream.use { inputStream ->
- MediaDownloaderHelper.getSplitElements(inputStream) { type, splitInputStream ->
- val fileName = "${type}_${Base64.UrlSafe.encode(protoMediaReference).replace("=", "")}"
- val bufferedInputStream = BufferedInputStream(splitInputStream)
- val fileType = MediaDownloaderHelper.getFileType(bufferedInputStream)
- val mediaFile = File(downloadMediaCacheFolder, "$fileName.${fileType.fileExtension}")
-
- FileOutputStream(mediaFile).use { fos ->
- bufferedInputStream.copyTo(fos)
- }
-
- mediaFiles[fileName] = fileType to mediaFile
- }
- }
- }
-
- updateProgress("downloaded")
- }.onFailure {
- printLog("failed to download media for ${message.messageDescriptor!!.conversationId}_${message.orderKey}")
- context.log.error("failed to download media for ${message.messageDescriptor!!.conversationId}_${message.orderKey}", it)
- }
- }
- }
- }
-
- threadPool.shutdown()
- threadPool.awaitTermination(30, TimeUnit.DAYS)
- processCount = 0
-
- printLog("writing downloaded medias...")
-
- //write the head of the html file
- output.write("""
- <!DOCTYPE html>
- <html>
- <head>
- <meta charset="UTF-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
- <title></title>
- </head>
- """.trimIndent().toByteArray())
-
- output.write("<!-- This file was generated by SnapEnhance ${BuildConfig.VERSION_NAME} -->\n".toByteArray())
-
- mediaFiles.forEach { (key, filePair) ->
- output.write("<div class=\"media-$key\"><!-- ".toByteArray())
- filePair.second.inputStream().use { inputStream ->
- val deflateInputStream = DeflaterInputStream(inputStream, Deflater(Deflater.BEST_COMPRESSION, true))
- (XposedHelpers.newInstance(
- Base64InputStream::class.java,
- deflateInputStream,
- android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
- true
- ) as InputStream).copyTo(output)
- }
- output.write(" --></div>\n".toByteArray())
- output.flush()
- updateProgress("wrote")
- }
- printLog("writing json conversation data...")
-
- //write the json file
- output.write("<script type=\"application/json\" class=\"exported_content\">".toByteArray())
-
- val b64os = (XposedHelpers.newInstance(
- Base64OutputStream::class.java,
- output,
- android.util.Base64.DEFAULT or android.util.Base64.NO_WRAP,
- true
- ) as OutputStream)
- val deflateOutputStream = DeflaterOutputStream(b64os, Deflater(Deflater.BEST_COMPRESSION, true), true)
- exportJson(deflateOutputStream)
- deflateOutputStream.finish()
- b64os.flush()
-
- output.write("</script>\n".toByteArray())
-
- printLog("writing template...")
-
- runCatching {
- ZipFile(context.bridgeClient.getApplicationApkPath()).use { apkFile ->
- //export rawinflate.js
- apkFile.getEntry("assets/web/rawinflate.js")?.let { entry ->
- output.write("<script>".toByteArray())
- apkFile.getInputStream(entry).copyTo(output)
- output.write("</script>\n".toByteArray())
- }
-
- //export avenir next font
- apkFile.getEntry("assets/web/avenir_next_medium.ttf")?.let { entry ->
- val encodedFontData = Base64.Default.encode(apkFile.getInputStream(entry).readBytes())
- output.write("""
- <style>
- @font-face {
- font-family: 'Avenir Next';
- src: url('data:font/truetype;charset=utf-8;base64, $encodedFontData');
- font-weight: normal;
- font-style: normal;
- }
- </style>
- """.trimIndent().toByteArray())
- }
-
- apkFile.getEntry("assets/web/export_template.html")?.let { entry ->
- apkFile.getInputStream(entry).copyTo(output)
- }
-
- apkFile.close()
- }
- }.onFailure {
- throw Throwable("Failed to read template from apk", it)
- }
-
- output.write("</html>".toByteArray())
- output.close()
- }
- }
-
- private fun exportJson(output: OutputStream) {
- val rootObject = JsonObject().apply {
- addProperty("conversationId", friendFeedEntry.key)
- addProperty("conversationName", friendFeedEntry.feedDisplayName)
-
- var index = 0
- val participants = mutableMapOf<String, Int>()
-
- add("participants", JsonObject().apply {
- conversationParticipants.forEach { (userId, friendInfo) ->
- add(userId, JsonObject().apply {
- addProperty("id", index)
- addProperty("displayName", friendInfo.displayName)
- addProperty("username", friendInfo.usernameForSorting)
- addProperty("bitmojiSelfieId", friendInfo.bitmojiSelfieId)
- })
- participants[userId] = index++
- }
- })
- add("messages", JsonArray().apply {
- messages.forEach { message ->
- add(JsonObject().apply {
- addProperty("orderKey", message.orderKey)
- addProperty("senderId", participants.getOrDefault(message.senderId.toString(), -1))
- addProperty("type", message.messageContent!!.contentType.toString())
-
- fun addUUIDList(name: String, list: List<SnapUUID>) {
- add(name, JsonArray().apply {
- list.map { participants.getOrDefault(it.toString(), -1) }.forEach { add(it) }
- })
- }
-
- addUUIDList("savedBy", message.messageMetadata!!.savedBy!!)
- addUUIDList("seenBy", message.messageMetadata!!.seenBy!!)
- addUUIDList("openedBy", message.messageMetadata!!.openedBy!!)
-
- add("reactions", JsonObject().apply {
- message.messageMetadata!!.reactions!!.forEach { reaction ->
- addProperty(
- participants.getOrDefault(reaction.userId.toString(), -1L).toString(),
- reaction.reactionId
- )
- }
- })
-
- addProperty("createdTimestamp", message.messageMetadata!!.createdAt)
- addProperty("readTimestamp", message.messageMetadata!!.readAt)
- addProperty("serializedContent", serializeMessageContent(message))
- addProperty("rawContent", Base64.UrlSafe.encode(message.messageContent!!.content!!))
-
- add("attachments", JsonArray().apply {
- MessageDecoder.decode(message.messageContent!!)
- .forEach attachments@{ attachments ->
- if (attachments.type == AttachmentType.STICKER) //TODO: implement stickers
- return@attachments
- add(JsonObject().apply {
- addProperty("key", attachments.mediaUrlKey?.replace("=", ""))
- addProperty("type", attachments.type.toString())
- add("encryption", attachments.attachmentInfo?.encryption?.let { encryption ->
- JsonObject().apply {
- addProperty("key", encryption.key)
- addProperty("iv", encryption.iv)
- }
- } ?: JsonNull.INSTANCE)
- })
- }
- })
- })
- }
- })
- }
-
- output.write(context.gson.toJson(rootObject).toByteArray())
- output.flush()
- }
-
- suspend fun exportTo(exportFormat: ExportFormat) {
- withContext(Dispatchers.IO) {
- FileOutputStream(outputFile).apply {
- when (exportFormat) {
- ExportFormat.HTML -> exportHtml(this)
- ExportFormat.JSON -> exportJson(this)
- ExportFormat.TEXT -> exportText(this)
- }
- close()
- }
- }
- }
-}-
\ No newline at end of file
diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/wrapper/impl/ConversationManager.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/wrapper/impl/ConversationManager.kt
@@ -3,6 +3,7 @@ package me.rhunk.snapenhance.core.wrapper.impl
import me.rhunk.snapenhance.common.data.MessageUpdate
import me.rhunk.snapenhance.core.ModContext
import me.rhunk.snapenhance.core.util.CallbackBuilder
+import me.rhunk.snapenhance.core.util.ktx.getObjectField
import me.rhunk.snapenhance.core.util.ktx.setObjectField
import me.rhunk.snapenhance.core.wrapper.AbstractWrapper
@@ -18,6 +19,7 @@ class ConversationManager(
private val fetchConversationWithMessagesPaginatedMethod by lazy { findMethodByName("fetchConversationWithMessagesPaginated") }
private val fetchConversationWithMessagesMethod by lazy { findMethodByName("fetchConversationWithMessages") }
private val fetchMessageByServerId by lazy { findMethodByName("fetchMessageByServerId") }
+ private val fetchMessagesByServerIds by lazy { findMethodByName("fetchMessagesByServerIds") }
private val displayedMessagesMethod by lazy { findMethodByName("displayedMessages") }
private val fetchMessage by lazy { findMethodByName("fetchMessage") }
@@ -105,4 +107,25 @@ class ConversationManager(
}.build()
)
}
+
+ fun fetchMessagesByServerIds(conversationId: String, serverMessageIds: List<Long>, onSuccess: (List<Message>) -> Unit, onError: (error: String) -> Unit) {
+ fetchMessagesByServerIds.invoke(
+ instanceNonNull(),
+ serverMessageIds.map {
+ CallbackBuilder.createEmptyObject(context.classCache.serverMessageIdentifier.constructors.first())?.apply {
+ setObjectField("mServerConversationId", conversationId.toSnapUUID().instanceNonNull())
+ setObjectField("mServerMessageId", it)
+ }
+ },
+ CallbackBuilder(context.mappings.getMappedClass("callbacks", "FetchMessagesByServerIdsCallback"))
+ .override("onSuccess") { param ->
+ onSuccess(param.arg<List<*>>(0).mapNotNull {
+ Message(it?.getObjectField("mMessage") ?: return@mapNotNull null)
+ })
+ }
+ .override("onError") {
+ onError(it.arg<Any>(0).toString())
+ }.build()
+ )
+ }
}
\ No newline at end of file
diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/wrapper/impl/Message.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/wrapper/impl/Message.kt
@@ -1,6 +1,8 @@
package me.rhunk.snapenhance.core.wrapper.impl
+import me.rhunk.snapenhance.common.data.ContentType
import me.rhunk.snapenhance.common.data.MessageState
+import me.rhunk.snapenhance.common.util.protobuf.ProtoReader
import me.rhunk.snapenhance.core.wrapper.AbstractWrapper
import org.mozilla.javascript.annotations.JSGetter
import org.mozilla.javascript.annotations.JSSetter
@@ -18,4 +20,8 @@ class Message(obj: Any?) : AbstractWrapper(obj) {
var messageMetadata by field("mMetadata") { MessageMetadata(it) }
@get:JSGetter @set:JSSetter
var messageState by enum("mState", MessageState.COMMITTED)
+
+ fun serialize() = if (messageContent!!.contentType == ContentType.CHAT) {
+ ProtoReader(messageContent!!.content!!).getString(2, 1) ?: "Failed to parse message"
+ } else null
}
\ No newline at end of file
