fix: fix AudioFrameObserver event bugs

android/build.gradle

@@ -47,7 +47,7 @@ dependencies {
   if (isDev(project)) {
     implementation fileTree(dir: "libs", include: ["*.jar"])
   } else {
-     api 'io.agora.rtc:iris-rtc:4.0.0-rc.1'
+     api 'io.agora.rtc:iris-rtc:4.0.0-rc.2'
      api 'io.agora.rtc:agora-special-full:4.0.0.5'
      implementation 'io.agora.rtc:full-screen-sharing:4.0.0.5'
   }

example/lib/examples/advanced/index.dart

@@ -25,6 +25,7 @@ import 'enable_virtualbackground/enable_virtualbackground.dart';
 import 'join_multiple_channel/join_multiple_channel.dart';
 import 'media_recorder/media_recorder.dart';
 import 'precall_test/precall_test.dart';
+import 'process_audio_raw_data/process_audio_raw_data.dart';
 import 'send_metadata/send_metadata.dart';
 import 'set_content_inspect/set_content_inspect.dart';
 import 'start_rhythm_player/start_rhythm_player.dart';
@@ -72,7 +73,7 @@ final advanced = [
       'widget': const StartLocalVideoTranscoder()
     },
   // {'name': 'ProcessVideoRawData', 'widget': const ProcessVideoRawData()},
-  // {'name': 'ProcessAudioRawData', 'widget': const ProcessAudioRawData()},
+  {'name': 'ProcessAudioRawData', 'widget': const ProcessAudioRawData()},
   {'name': 'AudioSpectrum', 'widget': const AudioSpectrum()},
   {'name': 'MediaRecorder', 'widget': const MediaRecorder()},
   {'name': 'PushVideoFrame', 'widget': const PushVideoFrame()},

example/lib/examples/advanced/process_audio_raw_data/process_audio_raw_data.dart

@@ -1,9 +1,13 @@
+import 'dart:io';
+
 import 'package:agora_rtc_engine/agora_rtc_engine.dart';
 import 'package:agora_rtc_engine_example/config/agora.config.dart' as config;
 import 'package:agora_rtc_engine_example/components/example_actions_widget.dart';
 import 'package:agora_rtc_engine_example/components/log_sink.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
+import 'package:path_provider/path_provider.dart';
+import 'package:path/path.dart' as path;
 
 /// ProcessVideoRawData Example
 class ProcessAudioRawData extends StatefulWidget {
@@ -24,6 +28,9 @@ class _State extends State<ProcessAudioRawData> {
   bool _isUseFlutterTexture = false;
   ChannelProfileType _channelProfileType =
       ChannelProfileType.channelProfileLiveBroadcasting;
+  late File _audioFile;
+  late File _playbackAudioFile;
+  late AudioFrameObserver _audioFrameObserver;
 
   @override
   void initState() {
@@ -40,6 +47,7 @@ class _State extends State<ProcessAudioRawData> {
   }
 
   Future<void> _dispose() async {
+    _stopAudioFrameRecord();
     await _engine.leaveChannel();
     await _engine.release();
   }
@@ -89,12 +97,30 @@ class _State extends State<ProcessAudioRawData> {
 
     await _engine.enableVideo();
 
-    _engine.getMediaEngine().registerAudioFrameObserver(AudioFrameObserver(
-      onRecordAudioFrame: (channelId, audioFrame) {
+    _audioFrameObserver = AudioFrameObserver(
+      onRecordAudioFrame: (channelId, audioFrame) async {
         debugPrint(
             '[onRecordAudioFrame] channelId: $channelId, audioFrame: ${audioFrame.toJson()}');
+        if (!isJoined) {
+          return;
+        }
+        if (audioFrame.buffer != null) {
+          await _audioFile.writeAsBytes(audioFrame.buffer!.toList(),
+              mode: FileMode.append, flush: true);
+        }
       },
-    ));
+      onPlaybackAudioFrame: (String channelId, AudioFrame audioFrame) async {
+        debugPrint(
+            '[onPlaybackAudioFrame] channelId: $channelId, audioFrame: ${audioFrame.toJson()}');
+        if (!isJoined) {
+          return;
+        }
+        if (audioFrame.buffer != null) {
+          await _playbackAudioFile.writeAsBytes(audioFrame.buffer!.toList(),
+              mode: FileMode.append, flush: true);
+        }
+      },
+    );
 
     await _engine.setVideoEncoderConfiguration(
       const VideoEncoderConfiguration(
@@ -106,11 +132,44 @@ class _State extends State<ProcessAudioRawData> {
 
     await _engine.startPreview();
 
+    await _startAudioFrameRecord();
+
     setState(() {
       _isReadyPreview = true;
     });
   }
 
+  Future<void> _startAudioFrameRecord() async {
+    Directory appDocDir = Platform.isAndroid
+        ? (await getExternalStorageDirectory())!
+        : await getApplicationDocumentsDirectory();
+
+    _audioFile = File(path.join(appDocDir.absolute.path, 'record_audio.raw'));
+    if (await _audioFile.exists()) {
+      await _audioFile.delete();
+    }
+    await _audioFile.create();
+    logSink
+        .log('onRecordAudioFrame file output to: ${_audioFile.absolute.path}');
+
+    _playbackAudioFile = File(path.join(
+      appDocDir.absolute.path,
+      'playback_audio.raw',
+    ));
+    if (await _playbackAudioFile.exists()) {
+      await _playbackAudioFile.delete();
+    }
+    await _playbackAudioFile.create();
+    logSink.log(
+        'onPlaybackAudioFrame file output to: ${_playbackAudioFile.absolute.path}');
+
+    _engine.getMediaEngine().registerAudioFrameObserver(_audioFrameObserver);
+  }
+
+  void _stopAudioFrameRecord() {
+    _engine.getMediaEngine().unregisterAudioFrameObserver(_audioFrameObserver);
+  }
+
   Future<void> _joinChannel() async {
     await _engine.joinChannel(
       token: config.token,
@@ -139,39 +198,8 @@ class _State extends State<ProcessAudioRawData> {
     return ExampleActionsWidget(
       displayContentBuilder: (context, isLayoutHorizontal) {
         if (!_isReadyPreview) return Container();
-        return Stack(
-          children: [
-            AgoraVideoView(
-              controller: VideoViewController(
-                rtcEngine: _engine,
-                canvas: const VideoCanvas(uid: 0),
-                useFlutterTexture: _isUseFlutterTexture,
-              ),
-            ),
-            Align(
-              alignment: Alignment.topLeft,
-              child: SingleChildScrollView(
-                scrollDirection: Axis.horizontal,
-                child: Row(
-                  children: List.of(remoteUid.map(
-                    (e) => SizedBox(
-                      width: 120,
-                      height: 120,
-                      child: AgoraVideoView(
-                        controller: VideoViewController.remote(
-                          rtcEngine: _engine,
-                          canvas: VideoCanvas(uid: e),
-                          connection:
-                              RtcConnection(channelId: _controller.text),
-                          useFlutterTexture: _isUseFlutterTexture,
-                        ),
-                      ),
-                    ),
-                  )),
-                ),
-              ),
-            )
-          ],
+        return const Center(
+          child: Text('No Preview'),
         );
       },
       actionsBuilder: (context, isLayoutHorizontal) {

ios/agora_rtc_engine.podspec

@@ -17,7 +17,7 @@ Pod::Spec.new do |s|
   s.source           = { :path => '.' }
   s.source_files = 'Classes/**/*.{h,mm,m,swift}'
   s.dependency 'Flutter'
-  s.dependency 'AgoraIrisRTC_iOS', '4.0.0-rc.1'
+  s.dependency 'AgoraIrisRTC_iOS', '4.0.0-rc.2'
   s.dependency 'AgoraRtcEngine_Special_iOS', '4.0.0.5'
   # s.dependency 'AgoraRtcWrapper'
   s.platform = :ios, '9.0'

lib/src/impl/agora_media_engine_impl_override.dart

@@ -5,13 +5,89 @@ import 'package:agora_rtc_engine/agora_rtc_engine.dart';
 import 'package:agora_rtc_engine/src/binding/agora_media_base_event_impl.dart';
 import 'package:agora_rtc_engine/src/binding/agora_media_engine_impl.dart'
     as media_engine_impl_binding;
+import 'package:agora_rtc_engine/src/binding/call_api_event_handler_buffer_ext.dart';
+import 'package:agora_rtc_engine/src/binding/event_handler_param_json.dart';
 import 'package:agora_rtc_engine/src/impl/agora_rtc_engine_impl.dart';
 import 'package:agora_rtc_engine/src/impl/api_caller.dart';
 import 'package:agora_rtc_engine/src/impl/disposable_object.dart';
 import 'package:iris_event/iris_event.dart';
 
 // ignore_for_file: public_member_api_docs, unused_local_variable
 
+class AudioFrameObserverWrapperOverride extends AudioFrameObserverWrapper {
+  AudioFrameObserverWrapperOverride(AudioFrameObserver audioFrameObserver)
+      : super(audioFrameObserver);
+
+  @override
+  void onEvent(String event, String data, List<Uint8List> buffers) {
+    if (!event.startsWith('AudioFrameObserver')) return;
+
+    final jsonMap = jsonDecode(data);
+    switch (event) {
+      case 'AudioFrameObserver_onPlaybackAudioFrameBeforeMixing':
+        if (audioFrameObserver.onPlaybackAudioFrameBeforeMixing == null) break;
+        AudioFrameObserverOnPlaybackAudioFrameBeforeMixingJson paramJson =
+            AudioFrameObserverOnPlaybackAudioFrameBeforeMixingJson.fromJson(
+                jsonMap);
+        paramJson = paramJson.fillBuffers(buffers);
+        String? channelId = paramJson.channelId;
+        int? uid = paramJson.uid;
+        AudioFrame? audioFrame = paramJson.audioFrame;
+        if (channelId == null || uid == null || audioFrame == null) {
+          break;
+        }
+        audioFrame = audioFrame.fillBuffers(buffers);
+        audioFrameObserver.onPlaybackAudioFrameBeforeMixing!(
+            channelId, uid, audioFrame);
+        break;
+
+      case 'AudioFrameObserver_onRecordAudioFrame':
+        if (audioFrameObserver.onRecordAudioFrame == null) break;
+        AudioFrameObserverBaseOnRecordAudioFrameJson paramJson =
+            AudioFrameObserverBaseOnRecordAudioFrameJson.fromJson(jsonMap);
+        paramJson = paramJson.fillBuffers(buffers);
+        String? channelId = paramJson.channelId;
+        AudioFrame? audioFrame = paramJson.audioFrame;
+        if (channelId == null || audioFrame == null) {
+          break;
+        }
+        audioFrame = audioFrame.fillBuffers(buffers);
+        audioFrameObserver.onRecordAudioFrame!(channelId, audioFrame);
+        break;
+
+      case 'AudioFrameObserver_onPlaybackAudioFrame':
+        if (audioFrameObserver.onPlaybackAudioFrame == null) break;
+        AudioFrameObserverBaseOnPlaybackAudioFrameJson paramJson =
+            AudioFrameObserverBaseOnPlaybackAudioFrameJson.fromJson(jsonMap);
+        paramJson = paramJson.fillBuffers(buffers);
+        String? channelId = paramJson.channelId;
+        AudioFrame? audioFrame = paramJson.audioFrame;
+        if (channelId == null || audioFrame == null) {
+          break;
+        }
+        audioFrame = audioFrame.fillBuffers(buffers);
+        audioFrameObserver.onPlaybackAudioFrame!(channelId, audioFrame);
+        break;
+
+      case 'AudioFrameObserver_onMixedAudioFrame':
+        if (audioFrameObserver.onMixedAudioFrame == null) break;
+        AudioFrameObserverBaseOnMixedAudioFrameJson paramJson =
+            AudioFrameObserverBaseOnMixedAudioFrameJson.fromJson(jsonMap);
+        paramJson = paramJson.fillBuffers(buffers);
+        String? channelId = paramJson.channelId;
+        AudioFrame? audioFrame = paramJson.audioFrame;
+        if (channelId == null || audioFrame == null) {
+          break;
+        }
+        audioFrame = audioFrame.fillBuffers(buffers);
+        audioFrameObserver.onMixedAudioFrame!(channelId, audioFrame);
+        break;
+      default:
+        break;
+    }
+  }
+}
+
 class MediaEngineImpl extends media_engine_impl_binding.MediaEngineImpl
     implements IrisEventHandler, AsyncDisposableObject {
   MediaEngineImpl._(this._rtcEngine) {
@@ -37,7 +113,7 @@ class MediaEngineImpl extends media_engine_impl_binding.MediaEngineImpl
             unregisterName: 'MediaEngine_unregisterAudioFrameObserver'),
         jsonEncode(param));
 
-    _eventHandlers.add(AudioFrameObserverWrapper(observer));
+    _eventHandlers.add(AudioFrameObserverWrapperOverride(observer));
   }
 
   @override
@@ -77,7 +153,7 @@ class MediaEngineImpl extends media_engine_impl_binding.MediaEngineImpl
             unregisterName: 'MediaEngine_unregisterAudioFrameObserver'),
         jsonEncode(param));
 
-    _eventHandlers.remove(AudioFrameObserverWrapper(observer));
+    _eventHandlers.remove(AudioFrameObserverWrapperOverride(observer));
   }
 
   @override

lib/src/impl/agora_media_recorder_impl_override.dart

@@ -60,8 +60,6 @@ class MediaRecorderImpl extends media_recorder_impl_binding.MediaRecorderImpl
           jsonEncode({}));
       _eventHandlers.clear();
     }
-
-    _rtcEngine.removeFromPool(MediaRecorderImpl);
   }
 
   @override

lib/src/impl/agora_rtc_engine_impl.dart

@@ -66,8 +66,9 @@ class ObjectPool {
   }
 
   Future<void> clear() async {
-    for (final key in pool.keys) {
-      await pool[key]?.disposeAsync();
+    final values = pool.values;
+    for (final v in values) {
+      await v.disposeAsync();
     }
 
     pool.clear();
@@ -244,7 +245,9 @@ class RtcEngineImpl extends rtc_engine_ex_binding.RtcEngineExImpl
         release(sync: true);
       },
     );
-    WidgetsBinding.instance.addObserver(_lifecycle!);
+    // Compatible with 2.10
+    // ignore: invalid_null_aware_operator
+    WidgetsBinding.instance?.addObserver(_lifecycle!);
 
     if (defaultTargetPlatform == TargetPlatform.android) {
       final externalFilesDir =
@@ -282,7 +285,9 @@ class RtcEngineImpl extends rtc_engine_ex_binding.RtcEngineExImpl
     if (_instance == null) return;
 
     if (_lifecycle != null) {
-      WidgetsBinding.instance.removeObserver(_lifecycle!);
+      // Compatible with 2.10
+      // ignore: invalid_null_aware_operator
+      WidgetsBinding.instance?.removeObserver(_lifecycle!);
       _lifecycle = null;
     }
 
@@ -1084,4 +1089,9 @@ class VideoDeviceManagerImpl extends rtc_engine_binding.VideoDeviceManagerImpl
   Future<void> release() async {
     _instance = null;
   }
+
+  @override
+  Future<void> startDeviceTest(int hwnd) {
+    throw AgoraRtcException(code: ErrorCodeType.errNotSupported.value());
+  }
 }

lib/src/impl/agora_spatial_audio_impl_override.dart

@@ -47,8 +47,6 @@ class LocalSpatialAudioEngineImpl extends spatial_audio_binding
     final param = createParams({});
     final callApiResult =
         await apiCaller.callIrisApi(apiType, jsonEncode(param), buffers: null);
-
-    _rtcEngine.removeFromPool(LocalSpatialAudioEngineImpl);
   }
 
   @override

lib/src/impl/api_caller.dart

@@ -635,6 +635,7 @@ class _ApiCallExecutorInternal implements _ApiCallExecutorBase {
                 ));
       } else if (key.op == CallIrisEventOp.dispose) {
         _irisEventHandlerObservers[key]?.dispose();
+        _irisEventHandlerObservers.remove(key);
       }
     } else if (key is IrisEventObserverKey) {
       if (key.op == CallIrisEventOp.create) {
@@ -650,6 +651,7 @@ class _ApiCallExecutorInternal implements _ApiCallExecutorBase {
                 ));
       } else if (key.op == CallIrisEventOp.dispose) {
         _irisEventHandlerObservers[key]?.dispose();
+        _irisEventHandlerObservers.remove(key);
       }
     }
 

macos/agora_rtc_engine.podspec

@@ -16,8 +16,8 @@ A new flutter plugin project.
   s.source_files = 'Classes/**/*.{h,mm}', 'Classes/File.swift'
   s.dependency 'FlutterMacOS'
   # s.dependency 'AgoraRtcWrapper'
-  s.dependency  'AgoraRtcEngine_Special_macOS', '4.0.0.5'
-  s.dependency 'AgoraIrisRTC_macOS', '4.0.0-rc.1'
+  s.dependency 'AgoraRtcEngine_Special_macOS', '4.0.0.5'
+  s.dependency 'AgoraIrisRTC_macOS', '4.0.0-rc.2'
 
   s.platform = :osx, '10.11'
   s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' }

pubspec.yaml

@@ -21,7 +21,7 @@ dependencies:
 dev_dependencies:
   flutter_test:
     sdk: flutter
-  flutter_lints: ^2.0.0
+  flutter_lints: ^1.0.0
   build_runner: ^2.1.7
   json_serializable: ^6.1.1
   ffigen: '>=4.1.2'