commit 4b93ec7181d02dd11d27039adea082ac91870eaa parent 95eb350066c6087eb77c860e78e6505df7fafab2 Author: rhunk <101876869+rhunk@users.noreply.github.com> Date: Thu, 27 Jun 2024 15:41:08 +0200 feat(core): enhanced transcript in notifications Signed-off-by: rhunk <101876869+rhunk@users.noreply.github.com> Diffstat:
6 files changed, 49 insertions(+), 18 deletions(-)
diff --git a/common/src/main/assets/lang/en_US.json b/common/src/main/assets/lang/en_US.json @@ -1074,6 +1074,10 @@ "enhanced_transcript": { "name": "Enhanced Transcript", "description": "Improves the voice note transcript using DeepL.\nBefore using this feature, please ensure that you have read their privacy policy." + }, + "enhanced_transcript_in_notifications": { + "name": "Enhanced Transcript in Notifications", + "description": "Transcribes voice notes in notifications using DeepL. This requires the Chat Preview feature to be enabled in Better Notifications" } } }, diff --git a/common/src/main/kotlin/me/rhunk/snapenhance/common/config/impl/Experimental.kt b/common/src/main/kotlin/me/rhunk/snapenhance/common/config/impl/Experimental.kt @@ -21,6 +21,7 @@ class Experimental : ConfigContainer() { val forceTranscription = boolean("force_transcription") { requireRestart() } val preferredTranscriptionLang = string("preferred_transcription_lang") { requireRestart() } val enhancedTranscript = boolean("enhanced_transcript") { requireRestart(); addNotices(FeatureNotice.UNSTABLE) } + val enhancedTranscriptInNotifications = boolean("enhanced_transcript_in_notifications") { requireRestart(); addNotices(FeatureNotice.UNSTABLE) } } class ComposerHooksConfig: ConfigContainer(hasGlobalState = true) { diff --git a/common/src/main/kotlin/me/rhunk/snapenhance/common/util/snap/RemoteMediaResolver.kt b/common/src/main/kotlin/me/rhunk/snapenhance/common/util/snap/RemoteMediaResolver.kt @@ -14,7 +14,7 @@ object RemoteMediaResolver { private val urlCache = mutableMapOf<String, String>() - private val okHttpClient = OkHttpClient.Builder() + val okHttpClient = OkHttpClient.Builder() .followRedirects(true) .retryOnConnectionFailure(true) .readTimeout(20, java.util.concurrent.TimeUnit.SECONDS) @@ -36,7 +36,7 @@ object RemoteMediaResolver { } .build() - private fun newResolveRequest(protoKey: ByteArray) = Request.Builder() + fun newResolveRequest(protoKey: ByteArray) = Request.Builder() .url(BOLT_HTTP_RESOLVER_URL + "/resolve?co=" + Base64.getUrlEncoder().encodeToString(protoKey)) .addHeader("User-Agent", Constants.USER_AGENT) .build() @@ -54,7 +54,7 @@ object RemoteMediaResolver { } } - fun downloadBoltMedia(protoKey: ByteArray, decryptionCallback: (InputStream) -> InputStream = { it }, resultCallback: (stream: InputStream, length: Long) -> Unit) { + inline fun downloadBoltMedia(protoKey: ByteArray, decryptionCallback: (InputStream) -> InputStream = { it }, resultCallback: (stream: InputStream, length: Long) -> Unit) { okHttpClient.newCall(newResolveRequest(protoKey)).execute().use { response -> if (!response.isSuccessful) { throw Throwable("invalid response ${response.code}") diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/downloader/decoder/MessageDecoder.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/downloader/decoder/MessageDecoder.kt @@ -31,7 +31,7 @@ data class DecodedAttachment( } @OptIn(ExperimentalEncodingApi::class) - inline fun openStream(crossinline callback: (mediaStream: InputStream?, length: Long) -> Unit) { + inline fun openStream(callback: (mediaStream: InputStream?, length: Long) -> Unit) { boltKey?.let { mediaUrlKey -> RemoteMediaResolver.downloadBoltMedia(Base64.UrlSafe.decode(mediaUrlKey), decryptionCallback = { attachmentInfo?.encryption?.decryptInputStream(it) ?: it diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/experiments/BetterTranscript.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/experiments/BetterTranscript.kt @@ -16,13 +16,14 @@ import java.lang.reflect.Method import java.nio.ByteBuffer class BetterTranscript: Feature("Better Transcript", loadParams = FeatureLoadParams.ACTIVITY_CREATE_SYNC) { + val transcriptApi by lazy { TranscriptApi() } + override fun onActivityCreate() { if (context.config.experimental.betterTranscript.globalState != true) return val config = context.config.experimental.betterTranscript val preferredTranscriptionLang = config.preferredTranscriptionLang.getNullable()?.takeIf { it.isNotBlank() } - val transcriptApi by lazy { TranscriptApi() } if (config.forceTranscription.get()) { context.event.subscribe(BuildMessageEvent::class, priority = 104) { event -> diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/messaging/Notifications.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/messaging/Notifications.kt @@ -26,7 +26,9 @@ import me.rhunk.snapenhance.core.features.Feature import me.rhunk.snapenhance.core.features.FeatureLoadParams import me.rhunk.snapenhance.core.features.impl.FriendMutationObserver import me.rhunk.snapenhance.core.features.impl.downloader.MediaDownloader +import me.rhunk.snapenhance.core.features.impl.downloader.decoder.AttachmentType import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder +import me.rhunk.snapenhance.core.features.impl.experiments.BetterTranscript import me.rhunk.snapenhance.core.features.impl.spying.StealthMode import me.rhunk.snapenhance.core.util.hook.HookStage import me.rhunk.snapenhance.core.util.hook.hook @@ -34,6 +36,7 @@ import me.rhunk.snapenhance.core.util.ktx.setObjectField import me.rhunk.snapenhance.core.util.media.PreviewUtils import me.rhunk.snapenhance.core.wrapper.impl.Message import me.rhunk.snapenhance.core.wrapper.impl.SnapUUID +import okhttp3.RequestBody.Companion.toRequestBody import kotlin.coroutines.suspendCoroutine class Notifications : Feature("Notifications", loadParams = FeatureLoadParams.INIT_SYNC) { @@ -384,23 +387,45 @@ class Notifications : Feature("Notifications", loadParams = FeatureLoadParams.IN } if (config.chatPreview.get()) { - if (contentType == ContentType.CHAT) { - setNotificationText(message.serialize() ?: "[Failed to parse message]") + var isChatMessage = contentType == ContentType.CHAT + var serializedMessage = if (isChatMessage) { + message.serialize() ?: "[Failed to parse message]" } else { - if (config.stackedMediaMessages.get()) { - setNotificationText(buildString { - append("[") - append(context.translation.getCategory("content_type")[contentType.name]) - append("]") - if (config.mediaCaption.get()) { - message.serialize()?.let { append(" $it") } + "[${context.translation.getCategory("content_type")[contentType.name]}]${ + if (config.mediaCaption.get()) { + message.serialize() ?: "" + } else "" + }" + } + + if (contentType == ContentType.NOTE && context.config.experimental.betterTranscript.takeIf { it.globalState == true }?.enhancedTranscriptInNotifications?.get() == true) { + val transcriptApi = context.feature(BetterTranscript::class).transcriptApi + MessageDecoder.decode(message.messageContent!!).firstOrNull { it.type == AttachmentType.NOTE }?.also { media -> + runCatching { + media.openStream { mediaStream, length -> + if (mediaStream == null || length > 25 * 1024 * 1024) { + context.log.error("Failed to open media stream or media is too large") + return@openStream + } + val text = transcriptApi.transcribe( + mediaStream.readBytes().toRequestBody(), + lang = context.config.experimental.betterTranscript.preferredTranscriptionLang.getNullable()?.takeIf { it.isNotBlank() } + )?.takeIf { it.isNotBlank() } ?: return@openStream + serializedMessage = "\uD83C\uDFA4 $text" + isChatMessage = true } - }) - } else { - sendNotification(message, data, true) - return + }.onFailure { + context.log.error("Failed to transcribe message", it) + } } } + + if (isChatMessage || config.stackedMediaMessages.get()) { + setNotificationText(serializedMessage) + } else { + sendNotification(message, data, true) + return + } computeMessages() }