commit 56e434cb15307c6bf53a2acd7eb62e725c0902fb
parent dc5b91a42ad3a741d59ec010e4eb442a8d8e7800
Author: rhunk <101876869+rhunk@users.noreply.github.com>
Date: Sat, 5 Jul 2025 00:49:59 +0200
feat(core): notification transcript
Signed-off-by: rhunk <101876869+rhunk@users.noreply.github.com>
Diffstat:
4 files changed, 64 insertions(+), 4 deletions(-)
diff --git a/common/src/main/assets/lang/en_US.json b/common/src/main/assets/lang/en_US.json
@@ -1056,6 +1056,10 @@
"preferred_transcription_lang": {
"name": "Preferred Transcription Language",
"description": "The preferred language for the voice note transcript (e.g. EN, ES, FR)"
+ },
+ "notification_transcript": {
+ "name": "Notification Transcript",
+ "description": "Transcribes voice notes in notifications\nThis feature requires the Chat Preview feature to be enabled in Better Notifications"
}
}
},
diff --git a/common/src/main/kotlin/me/rhunk/snapenhance/common/config/impl/Experimental.kt b/common/src/main/kotlin/me/rhunk/snapenhance/common/config/impl/Experimental.kt
@@ -20,6 +20,7 @@ class Experimental : ConfigContainer() {
class BetterTranscriptConfig: ConfigContainer(hasGlobalState = true) {
val forceTranscription = boolean("force_transcription") { requireRestart() }
val preferredTranscriptionLang = string("preferred_transcription_lang") { requireRestart() }
+ val notificationTranscript = boolean("notification_transcript") { requireRestart() }
}
class ComposerHooksConfig: ConfigContainer(hasGlobalState = true) {
diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/experiments/BetterTranscript.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/experiments/BetterTranscript.kt
@@ -4,20 +4,47 @@ import me.rhunk.snapenhance.common.data.ContentType
import me.rhunk.snapenhance.common.util.protobuf.ProtoEditor
import me.rhunk.snapenhance.core.event.events.impl.BuildMessageEvent
import me.rhunk.snapenhance.core.features.Feature
+import me.rhunk.snapenhance.core.util.dataBuilder
import me.rhunk.snapenhance.core.util.hook.HookStage
import me.rhunk.snapenhance.core.util.hook.hook
import me.rhunk.snapenhance.core.util.ktx.getObjectFieldOrNull
import me.rhunk.snapenhance.core.util.ktx.setObjectField
+import java.nio.ByteBuffer
class BetterTranscript: Feature("Better Transcript") {
+ private val voiceML: Any by lazy {
+ findClass("com.snapchat.client.voiceml.IVoiceMLSDK").getMethod("create").invoke(null) ?: error("Failed to create IVoiceMLSDK instance")
+ }
+
+ private fun createAsrConfig(): Any? {
+ findClass("com.snapchat.client.voiceml.IConfigFactory").methods.first { it.name == "simpleAsrConfig" }.let { method ->
+ return method.invoke(null, method.parameterTypes[0].dataBuilder {
+ set("mSampleRate", 44100)
+ set("mLanguageModel", "en")
+ set("mUseCase", "VOICENOTESTRANSCRIPTION")
+ set("mAppVersion", "voice note transcript")
+ set("mUiLanguage", "en")
+ set("mAuthType", "SNAPTOKEN")
+ set("mEncoding", "AAC")
+ })
+ }
+ }
+
+ fun transcribe(audio: ByteBuffer): String? {
+ val transcribeMethod = voiceML.javaClass.methods.first { it.name == "asrTranscribe" }
+ val snapToken = context.database.getAccessTokens(context.database.myUserId)?.get("api-gateway") ?: error("Failed to get api-gateway token")
+
+ return transcribeMethod.invoke(voiceML, snapToken, createAsrConfig(), audio)
+ ?.let { asrResult ->
+ asrResult.getObjectFieldOrNull("mTranscription")?.toString()
+ }
+ }
+
override fun init() {
if (context.config.experimental.betterTranscript.globalState != true) return
onNextActivityCreate {
val config = context.config.experimental.betterTranscript
- val preferredTranscriptionLang = config.preferredTranscriptionLang.getNullable()?.takeIf {
- it.isNotBlank()
- }
if (config.forceTranscription.get()) {
context.event.subscribe(BuildMessageEvent::class, priority = 104) { event ->
@@ -33,7 +60,9 @@ class BetterTranscript: Feature("Better Transcript") {
}
findClass("com.snapchat.client.voiceml.IVoiceMLSDK\$CppProxy").hook("asrTranscribe", HookStage.BEFORE) { param ->
- preferredTranscriptionLang?.lowercase()?.let {
+ config.preferredTranscriptionLang.getNullable()?.takeIf {
+ it.isNotBlank()
+ }?.trim()?.lowercase()?.let {
val asrConfig = param.arg<Any>(1)
asrConfig.getObjectFieldOrNull("mBaseConfig")?.apply {
setObjectField("mLanguageModel", it)
diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/messaging/Notifications.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/messaging/Notifications.kt
@@ -24,7 +24,9 @@ import me.rhunk.snapenhance.core.event.events.impl.SnapWidgetBroadcastReceiveEve
import me.rhunk.snapenhance.core.features.Feature
import me.rhunk.snapenhance.core.features.impl.FriendMutationObserver
import me.rhunk.snapenhance.core.features.impl.downloader.MediaDownloader
+import me.rhunk.snapenhance.core.features.impl.downloader.decoder.AttachmentType
import me.rhunk.snapenhance.core.features.impl.downloader.decoder.MessageDecoder
+import me.rhunk.snapenhance.core.features.impl.experiments.BetterTranscript
import me.rhunk.snapenhance.core.features.impl.spying.StealthMode
import me.rhunk.snapenhance.core.util.hook.HookStage
import me.rhunk.snapenhance.core.util.hook.findRestrictedConstructor
@@ -34,6 +36,7 @@ import me.rhunk.snapenhance.core.util.ktx.setObjectField
import me.rhunk.snapenhance.core.util.media.PreviewUtils
import me.rhunk.snapenhance.core.wrapper.impl.Message
import me.rhunk.snapenhance.core.wrapper.impl.SnapUUID
+import java.nio.ByteBuffer
import kotlin.coroutines.suspendCoroutine
class Notifications : Feature("Notifications") {
@@ -387,6 +390,29 @@ class Notifications : Feature("Notifications") {
}"
}
+ if (contentType == ContentType.NOTE && context.config.experimental.betterTranscript.takeIf { it.globalState == true }?.notificationTranscript?.get() == true) {
+ MessageDecoder.decode(message.messageContent!!).firstOrNull { it.type == AttachmentType.NOTE }?.also { media ->
+ runCatching {
+ media.openStream { mediaStream, length ->
+ if (mediaStream == null || length > 25 * 1024 * 1024) {
+ context.log.error("Failed to open media stream or media is too large")
+ return@openStream
+ }
+
+ val text = context.feature(BetterTranscript::class).transcribe(
+ ByteBuffer.allocateDirect(length.toInt()).apply {
+ put(mediaStream.readBytes())
+ rewind()
+ }) ?: return@openStream
+ serializedMessage = "\uD83C\uDFA4 $text"
+ isChatMessage = true
+ }
+ }.onFailure {
+ context.log.error("Failed to transcribe message", it)
+ }
+ }
+ }
+
if (isChatMessage || config.stackedMediaMessages.get()) {
setNotificationText(serializedMessage)
} else {
