commit db2cc0d785a15ab333b32249c6278630d9910aa5 parent ec12980571f5e53aa503d2a49f41a48f260351a5 Author: rhunk <101876869+rhunk@users.noreply.github.com> Date: Fri, 8 Mar 2024 17:49:53 +0100 feat(experimental): audio call recorder Diffstat:
12 files changed, 288 insertions(+), 24 deletions(-)
diff --git a/app/src/main/kotlin/me/rhunk/snapenhance/download/DownloadProcessor.kt b/app/src/main/kotlin/me/rhunk/snapenhance/download/DownloadProcessor.kt @@ -254,7 +254,7 @@ class DownloadProcessor ( val outputFile = File.createTempFile("voice_note", ".$format") newFFMpegProcessor(pendingTask).execute(FFMpegProcessor.Request( action = FFMpegProcessor.Action.AUDIO_CONVERSION, - inputs = listOf(media.file), + inputs = listOf(media.file.absolutePath), output = outputFile )) media.file.delete() @@ -291,7 +291,7 @@ class DownloadProcessor ( runCatching { newFFMpegProcessor(pendingTask).execute(FFMpegProcessor.Request( action = FFMpegProcessor.Action.DOWNLOAD_DASH, - inputs = listOf(dashPlaylistFile), + inputs = listOf(dashPlaylistFile.absolutePath), output = outputFile, startTime = dashOptions.offsetTime, duration = dashOptions.duration @@ -357,6 +357,22 @@ class DownloadProcessor ( } runCatching { + if (downloadRequest.isAudioStream) { + val streamUrl = downloadRequest.inputMedias.first().content + val outputFile = File.createTempFile("audio_stream", ".mp3") + + callbackOnProgress("Downloading audio stream") + pendingTask.updateProgress("Downloading audio stream") + newFFMpegProcessor(pendingTask).execute(FFMpegProcessor.Request( + action = FFMpegProcessor.Action.DOWNLOAD_AUDIO_STREAM, + inputs = listOf(streamUrl), + output = outputFile, + audioStreamFormat = downloadRequest.audioStreamFormat + )) + saveMediaToGallery(pendingTask, outputFile, downloadMetadata) + return@launch + } + //first download all input medias into cache val downloadedMedias = downloadInputMedias(pendingTask, downloadRequest).map { it.key to DownloadedFile(it.value, FileType.fromFile(it.value)) @@ -406,7 +422,7 @@ class DownloadProcessor ( newFFMpegProcessor(pendingTask).execute(FFMpegProcessor.Request( action = FFMpegProcessor.Action.MERGE_OVERLAY, - inputs = listOf(renamedMedia), + inputs = listOf(renamedMedia.absolutePath), output = mergedOverlay, overlay = renamedOverlayMedia )) diff --git a/app/src/main/kotlin/me/rhunk/snapenhance/download/FFMpegProcessor.kt b/app/src/main/kotlin/me/rhunk/snapenhance/download/FFMpegProcessor.kt @@ -1,5 +1,6 @@ package me.rhunk.snapenhance.download +import android.media.AudioFormat import android.media.MediaMetadataRetriever import com.arthenica.ffmpegkit.FFmpegKit import com.arthenica.ffmpegkit.FFmpegSession @@ -9,6 +10,7 @@ import kotlinx.coroutines.suspendCancellableCoroutine import me.rhunk.snapenhance.LogManager import me.rhunk.snapenhance.RemoteSideContext import me.rhunk.snapenhance.common.config.impl.DownloaderConfig +import me.rhunk.snapenhance.common.data.download.AudioStreamFormat import me.rhunk.snapenhance.common.logger.LogLevel import me.rhunk.snapenhance.task.PendingTask import java.io.File @@ -62,16 +64,18 @@ class FFMpegProcessor( DOWNLOAD_DASH, MERGE_OVERLAY, AUDIO_CONVERSION, - MERGE_MEDIA + MERGE_MEDIA, + DOWNLOAD_AUDIO_STREAM, } data class Request( val action: Action, - val inputs: List<File>, + val inputs: List<String>, val output: File, val overlay: File? = null, //only for MERGE_OVERLAY val startTime: Long? = null, //only for DOWNLOAD_DASH - val duration: Long? = null //only for DOWNLOAD_DASH + val duration: Long? = null, //only for DOWNLOAD_DASH + val audioStreamFormat: AudioStreamFormat? = null, //only for DOWNLOAD_AUDIO_STREAM ) @@ -113,8 +117,8 @@ class FFMpegProcessor( } val inputArguments = ArgumentList().apply { - args.inputs.forEach { file -> - this += "-i" to file.absolutePath + args.inputs.forEach { path -> + this += "-i" to path } } @@ -150,7 +154,7 @@ class FFMpegProcessor( inputArguments.clear() val filesInfo = args.inputs.mapNotNull { file -> runCatching { - MediaMetadataRetriever().apply { setDataSource(file.absolutePath) } + MediaMetadataRetriever().apply { setDataSource(file) } }.getOrNull()?.let { file to it } } @@ -173,7 +177,7 @@ class FFMpegProcessor( containsNoSound = true filterSecondPart.append("[v$index][${filesInfo.size}]") } - inputArguments += "-i" to file.absolutePath + inputArguments += "-i" to file } if (containsNoSound) { @@ -194,6 +198,18 @@ class FFMpegProcessor( filesInfo.forEach { it.second.close() } } + Action.DOWNLOAD_AUDIO_STREAM -> { + outputArguments.clear() + globalArguments += "-f" to when (args.audioStreamFormat!!.encoding) { + AudioFormat.ENCODING_PCM_8BIT -> "u8" + AudioFormat.ENCODING_PCM_16BIT -> "s16le" + AudioFormat.ENCODING_PCM_FLOAT -> "f32le" + AudioFormat.ENCODING_PCM_32BIT -> "s32le" + else -> throw IllegalArgumentException("Unsupported audio encoding") + } + globalArguments += "-ar" to args.audioStreamFormat!!.sampleRate.toString() + globalArguments += "-ac" to args.audioStreamFormat!!.channels.toString() + } } outputArguments += args.output.absolutePath newFFMpegTask(globalArguments, inputArguments, outputArguments) diff --git a/app/src/main/kotlin/me/rhunk/snapenhance/ui/manager/pages/TasksRoot.kt b/app/src/main/kotlin/me/rhunk/snapenhance/ui/manager/pages/TasksRoot.kt @@ -101,7 +101,7 @@ class TasksRoot : Routes.Route() { runCatching { context.shortToast("Merging ${filesToMerge.size} files") FFMpegProcessor.newFFMpegProcessor(context, pendingTask).execute( - FFMpegProcessor.Request(FFMpegProcessor.Action.MERGE_MEDIA, filesToMerge, mergedFile) + FFMpegProcessor.Request(FFMpegProcessor.Action.MERGE_MEDIA, filesToMerge.map { it.absolutePath }, mergedFile) ) DownloadProcessor(context, object: DownloadCallback.Default() { override fun onSuccess(outputPath: String) { diff --git a/common/src/main/assets/lang/en_US.json b/common/src/main/assets/lang/en_US.json @@ -686,6 +686,10 @@ "name": "Story Logger", "description": "Provides a history of friends stories" }, + "call_recorder": { + "name": "Call Recorder", + "description": "Automatically records audio calls" + }, "app_passcode": { "name": "App Passcode", "description": "Sets a passcode to lock the app" @@ -969,7 +973,8 @@ "profile_picture": "Profile Picture", "story_logger": "Story Logger", "message_logger": "Message Logger", - "merged": "Merged" + "merged": "Merged", + "voice_call": "Voice Call" }, "chat_action_menu": { diff --git a/common/src/main/kotlin/me/rhunk/snapenhance/common/config/impl/Experimental.kt b/common/src/main/kotlin/me/rhunk/snapenhance/common/config/impl/Experimental.kt @@ -24,6 +24,7 @@ class Experimental : ConfigContainer() { val convertMessageLocally = boolean("convert_message_locally") { requireRestart() } val newChatActionMenu = boolean("new_chat_action_menu") { requireRestart() } val storyLogger = boolean("story_logger") { requireRestart(); addNotices(FeatureNotice.UNSTABLE); } + val callRecorder = boolean("call_recorder") { requireRestart(); addNotices(FeatureNotice.UNSTABLE); } val appPasscode = string("app_passcode") val appLockOnResume = boolean("app_lock_on_resume") val infiniteStoryBoost = boolean("infinite_story_boost") diff --git a/common/src/main/kotlin/me/rhunk/snapenhance/common/data/download/DownloadRequest.kt b/common/src/main/kotlin/me/rhunk/snapenhance/common/data/download/DownloadRequest.kt @@ -6,6 +6,8 @@ import java.util.Locale data class DashOptions(val offsetTime: Long, val duration: Long?) +data class AudioStreamFormat(val channels: Int, val sampleRate: Int, val encoding: Int) + data class InputMedia( val content: String, val type: DownloadMediaType, @@ -17,18 +19,23 @@ data class InputMedia( class DownloadRequest( val inputMedias: Array<InputMedia>, val dashOptions: DashOptions? = null, + val audioStreamFormat: AudioStreamFormat? = null, private val flags: Int = 0, ) { object Flags { const val MERGE_OVERLAY = 1 - const val IS_DASH_PLAYLIST = 2 + const val DASH_PLAYLIST = 2 + const val AUDIO_STREAM = 4 } val isDashPlaylist: Boolean - get() = flags and Flags.IS_DASH_PLAYLIST != 0 + get() = flags and Flags.DASH_PLAYLIST != 0 val shouldMergeOverlay: Boolean get() = flags and Flags.MERGE_OVERLAY != 0 + + val isAudioStream: Boolean + get() = flags and Flags.AUDIO_STREAM != 0 } fun String.sanitizeForPath(): String { diff --git a/common/src/main/kotlin/me/rhunk/snapenhance/common/data/download/MediaDownloadSource.kt b/common/src/main/kotlin/me/rhunk/snapenhance/common/data/download/MediaDownloadSource.kt @@ -16,7 +16,8 @@ enum class MediaDownloadSource( PROFILE_PICTURE("profile_picture", "profile_picture"), STORY_LOGGER("story_logger", "story_logger"), MESSAGE_LOGGER("message_logger", "message_logger"), - MERGED("merged", "merged"); + MERGED("merged", "merged"), + VOICE_CALL("voice_call", "voice_call"); fun matches(source: String?): Boolean { if (source == null) return false diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/DownloadManagerClient.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/DownloadManagerClient.kt @@ -31,7 +31,7 @@ class DownloadManagerClient ( ) ), dashOptions = DashOptions(offsetTime, duration), - flags = DownloadRequest.Flags.IS_DASH_PLAYLIST + flags = DownloadRequest.Flags.DASH_PLAYLIST ) ) } @@ -67,4 +67,22 @@ class DownloadManagerClient ( ) ) } + + fun downloadStream( + streamUrl: String, + audioStreamFormat: AudioStreamFormat + ) { + enqueueDownloadRequest( + DownloadRequest( + inputMedias = arrayOf( + InputMedia( + content = streamUrl, + type = DownloadMediaType.REMOTE_MEDIA + ) + ), + flags = DownloadRequest.Flags.AUDIO_STREAM, + audioStreamFormat = audioStreamFormat + ) + ) + } } \ No newline at end of file diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/downloader/MediaDownloader.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/downloader/MediaDownloader.kt @@ -76,7 +76,7 @@ class MediaDownloader : MessagingRuleFeature("MediaDownloader", MessagingRuleTyp context.translation.getCategory("download_processor") } - private fun provideDownloadManagerClient( + fun provideDownloadManagerClient( mediaIdentifier: String, mediaAuthor: String, creationTimestamp: Long? = null, diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/experiments/CallRecorder.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/experiments/CallRecorder.kt @@ -0,0 +1,133 @@ +package me.rhunk.snapenhance.core.features.impl.experiments + +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioTrack +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import me.rhunk.snapenhance.common.data.download.AudioStreamFormat +import me.rhunk.snapenhance.common.data.download.MediaDownloadSource +import me.rhunk.snapenhance.core.features.Feature +import me.rhunk.snapenhance.core.features.FeatureLoadParams +import me.rhunk.snapenhance.core.features.impl.downloader.MediaDownloader +import me.rhunk.snapenhance.core.util.hook.HookStage +import me.rhunk.snapenhance.core.util.hook.hook +import me.rhunk.snapenhance.core.util.hook.hookConstructor +import me.rhunk.snapenhance.core.util.ktx.getObjectField +import me.rhunk.snapenhance.core.util.ktx.getObjectFieldOrNull +import me.rhunk.snapenhance.core.util.media.HttpServer +import java.io.PipedInputStream +import java.io.PipedOutputStream +import java.nio.ByteBuffer +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList + +class CallRecorder : Feature("Call Recorder", loadParams = FeatureLoadParams.INIT_SYNC) { + private val httpServer = HttpServer( + timeout = Integer.MAX_VALUE + ) + + override fun init() { + if (!context.config.experimental.callRecorder.get()) return + + val streamHandlers = ConcurrentHashMap<Int, MutableList<(data: ByteArray) -> Unit>>() // audioTrack -> handlers + val participants = CopyOnWriteArrayList<String>() + + findClass("com.snapchat.talkcorev3.CallingSessionState").hookConstructor(HookStage.AFTER) { param -> + val instance = param.thisObject<Any>() + val callingState = instance.getObjectFieldOrNull("mLocalUser")?.getObjectField("mCallingState") + + if (callingState.toString() == "IN_CALL") { + participants.clear() + participants.addAll((instance.getObjectField("mParticipants") as Map<*, *>).keys.map { it.toString() }) + } + } + + AudioTrack::class.java.apply { + getConstructor( + AudioAttributes::class.java, + AudioFormat::class.java, + Int::class.javaPrimitiveType, + Int::class.javaPrimitiveType, + Int::class.javaPrimitiveType, + ).hook(HookStage.BEFORE) { param -> + val audioAttributes = param.arg<AudioAttributes>(0) + if (audioAttributes.usage != AudioAttributes.USAGE_VOICE_COMMUNICATION) return@hook + val audioFormat = param.arg<AudioFormat>(1) + val hashCode = param.thisObject<Any>().hashCode() + + lateinit var streamUrl: String + streamUrl = httpServer.ensureServerStarted()?.putContent( + object: HttpServer.HttpContent() { + override val contentType: String = "audio/wav" + override val chunked: Boolean = true + override val contentLength: Long? = null + override val newBody: () -> HttpServer.HttpBody = { + object: HttpServer.HttpBody() { + val outputStream = PipedOutputStream() + val inputStream = PipedInputStream(outputStream) + + val handler: (byteArray: ByteArray) -> Unit = handler@{ byteArray -> + if (byteArray.isEmpty()) { + httpServer.removeUrl(streamUrl) + return@handler + } + runCatching { + outputStream.write(byteArray) + outputStream.flush() + }.onFailure { + context.log.warn("Failed to write to streaming url ${it.localizedMessage}") + } + } + + override val onOpen: () -> Unit = { + streamHandlers.getOrPut(hashCode) { CopyOnWriteArrayList() }.add(handler) + } + + override val readBytes: (byteArray: ByteArray) -> Int = { byteArray -> + runBlocking { + withTimeoutOrNull(3000L) { + inputStream.read(byteArray) + } ?: -1 + } + } + + override val onClose: () -> Unit = { + context.log.verbose("Streaming url closed") + streamHandlers[hashCode]?.remove(handler) + outputStream.close() + inputStream.close() + } + } + } + } + ) ?: return@hook + + context.log.verbose("streaming url = $streamUrl, sampleRate = ${audioFormat.sampleRate}, audioFormat = ${audioFormat.encoding}") + + context.feature(MediaDownloader::class).provideDownloadManagerClient( + UUID.randomUUID().toString(), + participants.mapNotNull { context.database.getFriendInfo(it)?.mutableUsername }.joinToString("-"), + System.currentTimeMillis(), + MediaDownloadSource.VOICE_CALL + ).downloadStream(streamUrl, AudioStreamFormat(audioFormat.channelCount, audioFormat.sampleRate, audioFormat.encoding)) + } + + getMethod("write", ByteBuffer::class.java, Int::class.javaPrimitiveType, Int::class.javaPrimitiveType).hook(HookStage.BEFORE) { param -> + streamHandlers[param.thisObject<Any>().hashCode()]?.let { handlers -> + val byteBuffer = param.arg<ByteBuffer>(0) + val position = byteBuffer.position() + val buffer = ByteArray(param.arg(1)) + byteBuffer.get(buffer) + byteBuffer.position(position) + handlers.forEach { it(buffer) } + } + } + + hook("release", HookStage.BEFORE) { + streamHandlers.remove(it.thisObject<Any>().hashCode())?.forEach { it(ByteArray(0)) } + } + } + } +}+ \ No newline at end of file diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/manager/impl/FeatureManager.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/manager/impl/FeatureManager.kt @@ -120,6 +120,7 @@ class FeatureManager( DisablePermissionRequests(), SessionEvents(), DefaultVolumeControls(), + CallRecorder(), ) initializeFeatures() diff --git a/core/src/main/kotlin/me/rhunk/snapenhance/core/util/media/HttpServer.kt b/core/src/main/kotlin/me/rhunk/snapenhance/core/util/media/HttpServer.kt @@ -27,7 +27,20 @@ class HttpServer( private var timeoutJob: Job? = null private var socketJob: Job? = null - private val cachedData = ConcurrentHashMap<String, Pair<InputStream, Long>>() + abstract class HttpBody { + abstract val readBytes: (byteArray: ByteArray) -> Int + open val onOpen: () -> Unit = {} + open val onClose: () -> Unit = {} + } + + abstract class HttpContent { + abstract val contentType: String + abstract val chunked: Boolean + abstract val contentLength: Long? + abstract val newBody: () -> HttpBody + } + + private val cachedData = ConcurrentHashMap<String, HttpContent>() private var serverSocket: ServerSocket? = null fun ensureServerStarted(): HttpServer? { @@ -83,15 +96,39 @@ class HttpServer( } fun close() { - serverSocket?.close() + runCatching { + serverSocket?.close() + } } fun putDownloadableContent(inputStream: InputStream, size: Long): String { val key = System.nanoTime().toString(16) - cachedData[key] = inputStream to size + cachedData[key] = object : HttpContent() { + override val contentType: String = "application/octet-stream" + override val chunked: Boolean = false + override val contentLength: Long = size + override val newBody: () -> HttpBody = { + object : HttpBody() { + override val readBytes: (byteArray: ByteArray) -> Int = { byteArray -> + inputStream.read(byteArray) + } + } + } + } + return "http://127.0.0.1:$port/$key" + } + + fun putContent(httpContent: HttpContent): String { + val key = System.nanoTime().toString(16) + cachedData[key] = httpContent return "http://127.0.0.1:$port/$key" } + fun removeUrl(url: String) { + val key = url.substringAfterLast('/') + cachedData.remove(key) + } + private fun handleRequest(socket: Socket) { val reader = BufferedReader(InputStreamReader(socket.getInputStream())) val outputStream = socket.getOutputStream() @@ -138,13 +175,41 @@ class HttpServer( with(writer) { println("HTTP/1.1 200 OK") println("Content-type: " + "application/octet-stream") - println("Content-length: " + requestedData.second) + if (requestedData.chunked) println("Transfer-encoding: chunked") + else println("Content-length: " + requestedData.contentLength) println() flush() } - cachedData.remove(fileRequested) - requestedData.first.use { - it.copyTo(outputStream) + + val responseBody = requestedData.newBody() + responseBody.onOpen() + try { + if (requestedData.chunked) { + val buffer = ByteArray(32768) + while (true) { + val read = responseBody.readBytes(buffer) + if (read == -1) break + outputStream.write(Integer.toHexString(read).toByteArray()) + outputStream.write("\r\n".toByteArray()) + outputStream.write(buffer, 0, read) + outputStream.write("\r\n".toByteArray()) + outputStream.flush() + } + } else { + cachedData.remove(fileRequested) + val buffer = ByteArray(4096) + while (true) { + val read = responseBody.readBytes(buffer) + if (read == -1) break + outputStream.write(buffer, 0, read) + outputStream.flush() + } + } + } catch (t: Throwable) { + AbstractLogger.directDebug("failed to write to socket ${t.localizedMessage}") + } finally { + if (requestedData.chunked) runCatching { outputStream.write("0\r\n\r\n".toByteArray()) } + responseBody.onClose() } outputStream.flush() close()