✨ Add waveform extraction natively part-1

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.0
+
+- Reworked waveforms from audio file
+  - Breaking: removed `readingComplete` PlayerState and `visualizerHeight`. With this, added `extractWaveforms` function to extract waveforms.
+  - Added `onCurrentExtractedWaveformData` and `onExtractionProgress` to monitor progress and currently extracted waveform data.
+
 ## 0.1.6
 
 - Fixed [#101](https://github.com/SimformSolutionsPvtLtd/audio_waveforms/issues/101) - Fixed setting volume for android throws error

android/build.gradle

@@ -41,7 +41,7 @@ android {
     }
 
     defaultConfig {
-        minSdkVersion 16
+        minSdkVersion 21
     }
 }
 

android/src/main/kotlin/com/simform/audio_waveforms/AudioPlayer.kt

@@ -1,15 +1,21 @@
 package com.simform.audio_waveforms
 
 import android.content.Context
+import android.os.Build
 import android.os.Handler
 import android.os.Looper
+import androidx.annotation.RequiresApi
 import com.google.android.exoplayer2.ExoPlayer
 import com.google.android.exoplayer2.MediaItem
 import com.google.android.exoplayer2.Player
 import io.flutter.plugin.common.MethodChannel
 import java.lang.Exception
 
-class AudioPlayer(context: Context, channel: MethodChannel, playerKey: String) {
+class AudioPlayer(
+    context: Context,
+    channel: MethodChannel,
+    playerKey: String
+) {
     private var handler: Handler = Handler(Looper.getMainLooper())
     private var runnable: Runnable? = null
     private var methodChannel = channel
@@ -19,6 +25,43 @@ class AudioPlayer(context: Context, channel: MethodChannel, playerKey: String) {
     private var isPlayerPrepared: Boolean = false
     private var finishMode = FinishMode.Stop
     private var key = playerKey
+    private var waveformExtractor: WaveformExtractor? = null
+    private var noOfSamples = 100
+
+    fun extractWaveform(
+        result: MethodChannel.Result,
+        path: String?,
+        noOfSamples: Int?,
+    ) {
+        if (path != null) {
+            this.noOfSamples = noOfSamples ?: 100
+            try {
+                waveformExtractor = WaveformExtractor(
+                    path = path,
+                    expectedPoints = this.noOfSamples,
+                    key = key,
+                    methodChannel = methodChannel,
+                    result = result,
+                    object : ExtractorCallBack {
+                        override fun onProgress(value: Float) {
+                            if (value == 1.0F) {
+                                result.success(waveformExtractor?.sampleData)
+                            }
+                        }
+                    }
+                )
+                waveformExtractor?.startDecode()
+                waveformExtractor?.stop()
+            } catch (e: Exception) {
+                result.error(
+                    Constants.LOG_TAG,
+                    "Can not extract waveform data from provided audio file path",
+                    e.toString()
+                )
+            }
+
+        }
+    }
 
     fun preparePlayer(
         result: MethodChannel.Result,

android/src/main/kotlin/com/simform/audio_waveforms/AudioWaveformsPlugin.kt

@@ -139,6 +139,16 @@ class AudioWaveformsPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
                     result.error(Constants.LOG_TAG, "Player key can't be null", "")
                 }
             }
+            Constants.extractWaveformData -> {
+                val key = call.argument(Constants.playerKey) as String?
+                val path = call.argument(Constants.path) as String?
+                val noOfSample = call.argument(Constants.noOfSamples) as Int?
+                if (key != null) {
+                    audioPlayers[key]?.extractWaveform(result, path, noOfSample)
+                } else {
+                    result.error(Constants.LOG_TAG, "Player key can't be null", "")
+                }
+            }
             Constants.stopAllPlayers -> {
                 for ((key, _) in audioPlayers) {
                     audioPlayers[key]?.stop(result)
@@ -198,7 +208,11 @@ class AudioWaveformsPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
 
     private fun initPlayer(playerKey: String) {
         if (audioPlayers[playerKey] == null) {
-            val newPlayer = AudioPlayer(applicationContext, channel, playerKey)
+            val newPlayer = AudioPlayer(
+                context = applicationContext,
+                channel = channel,
+                playerKey = playerKey,
+            )
             audioPlayers[playerKey] = newPlayer
         }
         return

android/src/main/kotlin/com/simform/audio_waveforms/Utils.kt

@@ -50,14 +50,18 @@ object Constants {
     const val volume = "volume"
     const val getDuration = "getDuration"
     const val durationType = "durationType"
-    const val seekToStart = "seekToStart"
     const val playerKey = "playerKey"
     const val current = "current"
     const val onCurrentDuration = "onCurrentDuration"
     const val stopAllPlayers = "stopAllPlayers"
     const val onDidFinishPlayingAudio = "onDidFinishPlayingAudio"
     const val finishMode = "finishMode"
     const val finishType = "finishType"
+    const val extractWaveformData = "extractWaveformData"
+    const val noOfSamples = "noOfSamples"
+    const val onCurrentExtractedWaveformData = "onCurrentExtractedWaveformData"
+    const val waveformData = "waveformData"
+    const val onExtractionProgressUpdate = "onExtractionProgressUpdate"
 }
 
 enum class FinishMode(val value:Int) {

android/src/main/kotlin/com/simform/audio_waveforms/WaveformExtractor.kt

@@ -0,0 +1,248 @@
+package com.simform.audio_waveforms
+
+import android.media.AudioFormat
+import android.media.MediaCodec
+import android.media.MediaExtractor
+import android.media.MediaFormat
+import android.os.Build
+import android.os.Handler
+import android.os.Looper
+import io.flutter.plugin.common.MethodChannel
+import java.nio.ByteBuffer
+import java.util.concurrent.CountDownLatch
+import kotlin.math.pow
+import kotlin.math.sqrt
+
+class WaveformExtractor(
+    private val path: String,
+    private val expectedPoints: Int,
+    private val key: String,
+    private val methodChannel: MethodChannel,
+    private val result: MethodChannel.Result,
+    private val extractorCallBack: ExtractorCallBack
+) {
+    private val handler = Handler(Looper.getMainLooper())
+    private var decoder: MediaCodec? = null
+    private var extractor: MediaExtractor? = null
+    private var duration = 0L
+    private var progress = 0F
+    private var currentProgress = 0F
+
+    @Volatile
+    private var started = false
+    private val finishCount = CountDownLatch(1)
+    private var inputEof = false
+    private var sampleRate = 0
+    private var channels = 1
+    private var pcmEncodingBit = 16
+    private var totalSamples = 0L
+    private var perSamplePoints = 0L
+
+    private fun getFormat(path: String): MediaFormat? {
+        val mediaExtractor = MediaExtractor()
+        this.extractor = mediaExtractor
+        mediaExtractor.setDataSource(path)
+        val trackCount = mediaExtractor.trackCount
+        repeat(trackCount) {
+            val format = mediaExtractor.getTrackFormat(it)
+            val mime = format.getString(MediaFormat.KEY_MIME) ?: ""
+            if (mime.contains("audio")) {
+                duration = format.getLong(MediaFormat.KEY_DURATION) / 1000000
+                mediaExtractor.selectTrack(it)
+                return format
+            }
+        }
+        return null
+    }
+
+    fun startDecode() {
+        try {
+            val format = getFormat(path) ?: error("No audio format found")
+            val mime = format.getString(MediaFormat.KEY_MIME) ?: error("No MIME type found")
+            decoder = MediaCodec.createDecoderByType(mime).also {
+                it.configure(format, null, null, 0)
+                it.setCallback(object : MediaCodec.Callback() {
+                    override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
+                        if (inputEof) return
+                        val extractor = extractor ?: return
+                        codec.getInputBuffer(index)?.let { buf ->
+                            val size = extractor.readSampleData(buf, 0)
+                            if (size > 0) {
+                                codec.queueInputBuffer(index, 0, size, extractor.sampleTime, 0)
+                                extractor.advance()
+                            } else {
+                                codec.queueInputBuffer(
+                                    index,
+                                    0,
+                                    0,
+                                    0,
+                                    MediaCodec.BUFFER_FLAG_END_OF_STREAM
+                                )
+                                inputEof = true
+                            }
+                        }
+                    }
+
+                    override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
+                        sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE)
+                        channels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
+                        pcmEncodingBit = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
+                            if (format.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
+                                when (format.getInteger(MediaFormat.KEY_PCM_ENCODING)) {
+                                    AudioFormat.ENCODING_PCM_16BIT -> 16
+                                    AudioFormat.ENCODING_PCM_8BIT -> 8
+                                    AudioFormat.ENCODING_PCM_FLOAT -> 32
+                                    else -> 16
+                                }
+                            } else {
+                                16
+                            }
+                        } else {
+                            16
+                        }
+                        totalSamples = sampleRate.toLong() * duration
+                        perSamplePoints = totalSamples / expectedPoints
+                    }
+
+                    override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
+                        result.error(
+                            Constants.LOG_TAG,
+                            e.message,
+                            "An error is thrown while decoding the audio file"
+                        )
+                        finishCount.countDown()
+                    }
+
+                    override fun onOutputBufferAvailable(
+                        codec: MediaCodec,
+                        index: Int,
+                        info: MediaCodec.BufferInfo
+                    ) {
+                        if (info.size > 0) {
+                            codec.getOutputBuffer(index)?.let { buf ->
+                                val size = info.size
+                                buf.position(info.offset)
+                                when (pcmEncodingBit) {
+                                    8 -> {
+                                        handle8bit(size, buf)
+                                    }
+                                    16 -> {
+                                        handle16bit(size, buf)
+                                    }
+                                    32 -> {
+                                        handle32bit(size, buf)
+                                    }
+                                }
+                                codec.releaseOutputBuffer(index, false)
+                            }
+                        }
+
+                        if (info.isEof()) {
+                            stop()
+                        }
+                    }
+
+                })
+                it.start()
+            }
+
+        } catch (e: Exception) {
+            result.error(
+                Constants.LOG_TAG,
+                e.message,
+                "An error is thrown before decoding the audio file"
+            )
+        }
+
+
+    }
+
+    var sampleData = ArrayList<Float>()
+    private var sampleCount = 0L
+    private var sampleSum = 0.0
+
+    private fun rms(value: Float) {
+        if (sampleCount == perSamplePoints) {
+            currentProgress++
+            progress = currentProgress / expectedPoints
+            val rms = sqrt(sampleSum / perSamplePoints)
+            sampleData.add(rms.toFloat())
+            extractorCallBack.onProgress(progress)
+            sampleCount = 0
+            sampleSum = 0.0
+
+            val args: MutableMap<String, Any?> = HashMap()
+            args[Constants.waveformData] = sampleData
+            args[Constants.progress] = progress
+            args[Constants.playerKey] = key
+            methodChannel.invokeMethod(
+                Constants.onCurrentExtractedWaveformData,
+                args
+            )
+        }
+
+        sampleCount++
+        sampleSum += value.toDouble().pow(2.0)
+    }
+
+    private fun handle8bit(size: Int, buf: ByteBuffer) {
+        repeat(size / if (channels == 2) 2 else 1) {
+            val result = buf.get().toInt() / 128f
+            if (channels == 2) {
+                buf.get()
+            }
+            rms(result)
+        }
+    }
+
+    private fun handle16bit(size: Int, buf: ByteBuffer) {
+        repeat(size / if (channels == 2) 4 else 2) {
+            val first = buf.get().toInt()
+            val second = buf.get().toInt() shl 8
+            val value = (first or second) / 32767f
+            if (channels == 2) {
+                buf.get()
+                buf.get()
+            }
+            rms(value)
+        }
+    }
+
+    private fun handle32bit(size: Int, buf: ByteBuffer) {
+        repeat(size / if (channels == 2) 8 else 4) {
+            val first = buf.get().toLong()
+            val second = buf.get().toLong() shl 8
+            val third = buf.get().toLong() shl 16
+            val forth = buf.get().toLong() shl 24
+            val value = (first or second or third or forth) / 2147483648f
+            if (channels == 2) {
+                buf.get()
+                buf.get()
+                buf.get()
+                buf.get()
+            }
+            rms(value)
+        }
+    }
+
+    fun stop() {
+        if (!started) return
+        started = false
+        decoder?.stop()
+        decoder?.release()
+        extractor?.release()
+        finishCount.countDown()
+    }
+
+    fun cancel() {
+        if (!started) return
+        handler.post { stop() }
+        finishCount.await()
+    }
+}
+
+fun MediaCodec.BufferInfo.isEof() = flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
+
+interface ExtractorCallBack {
+    fun onProgress(value: Float)
+}