From 7a8572804f0d242c7ead4c74458b04138be75980 Mon Sep 17 00:00:00 2001 From: littleGnAl Date: Fri, 16 Sep 2022 00:38:57 +0800 Subject: [PATCH] fix: fix AudioFrameObserver event bugs --- android/build.gradle | 2 +- example/lib/examples/advanced/index.dart | 3 +- .../process_audio_raw_data.dart | 100 +++++++++++------- ios/agora_rtc_engine.podspec | 2 +- .../agora_media_engine_impl_override.dart | 80 +++++++++++++- .../agora_media_recorder_impl_override.dart | 2 - lib/src/impl/agora_rtc_engine_impl.dart | 18 +++- .../agora_spatial_audio_impl_override.dart | 2 - lib/src/impl/api_caller.dart | 2 + macos/agora_rtc_engine.podspec | 4 +- pubspec.yaml | 2 +- scripts/flutter-build-runner.sh | 2 +- windows/CMakeLists.txt | 4 +- 13 files changed, 168 insertions(+), 55 deletions(-) diff --git a/android/build.gradle b/android/build.gradle index 784a6d4fe..b5a6b8c21 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -47,7 +47,7 @@ dependencies { if (isDev(project)) { implementation fileTree(dir: "libs", include: ["*.jar"]) } else { - api 'io.agora.rtc:iris-rtc:4.0.0-rc.1' + api 'io.agora.rtc:iris-rtc:4.0.0-rc.2' api 'io.agora.rtc:agora-special-full:4.0.0.5' implementation 'io.agora.rtc:full-screen-sharing:4.0.0.5' } diff --git a/example/lib/examples/advanced/index.dart b/example/lib/examples/advanced/index.dart index b25cdc879..5d30747bb 100644 --- a/example/lib/examples/advanced/index.dart +++ b/example/lib/examples/advanced/index.dart @@ -25,6 +25,7 @@ import 'enable_virtualbackground/enable_virtualbackground.dart'; import 'join_multiple_channel/join_multiple_channel.dart'; import 'media_recorder/media_recorder.dart'; import 'precall_test/precall_test.dart'; +import 'process_audio_raw_data/process_audio_raw_data.dart'; import 'send_metadata/send_metadata.dart'; import 'set_content_inspect/set_content_inspect.dart'; import 'start_rhythm_player/start_rhythm_player.dart'; @@ -72,7 +73,7 @@ final advanced = [ 'widget': const StartLocalVideoTranscoder() }, // {'name': 'ProcessVideoRawData', 'widget': const ProcessVideoRawData()}, - // {'name': 'ProcessAudioRawData', 'widget': const ProcessAudioRawData()}, + {'name': 'ProcessAudioRawData', 'widget': const ProcessAudioRawData()}, {'name': 'AudioSpectrum', 'widget': const AudioSpectrum()}, {'name': 'MediaRecorder', 'widget': const MediaRecorder()}, {'name': 'PushVideoFrame', 'widget': const PushVideoFrame()}, diff --git a/example/lib/examples/advanced/process_audio_raw_data/process_audio_raw_data.dart b/example/lib/examples/advanced/process_audio_raw_data/process_audio_raw_data.dart index 5389cc9ae..27411e1d6 100644 --- a/example/lib/examples/advanced/process_audio_raw_data/process_audio_raw_data.dart +++ b/example/lib/examples/advanced/process_audio_raw_data/process_audio_raw_data.dart @@ -1,9 +1,13 @@ +import 'dart:io'; + import 'package:agora_rtc_engine/agora_rtc_engine.dart'; import 'package:agora_rtc_engine_example/config/agora.config.dart' as config; import 'package:agora_rtc_engine_example/components/example_actions_widget.dart'; import 'package:agora_rtc_engine_example/components/log_sink.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:path/path.dart' as path; /// ProcessVideoRawData Example class ProcessAudioRawData extends StatefulWidget { @@ -24,6 +28,9 @@ class _State extends State { bool _isUseFlutterTexture = false; ChannelProfileType _channelProfileType = ChannelProfileType.channelProfileLiveBroadcasting; + late File _audioFile; + late File _playbackAudioFile; + late AudioFrameObserver _audioFrameObserver; @override void initState() { @@ -40,6 +47,7 @@ class _State extends State { } Future _dispose() async { + _stopAudioFrameRecord(); await _engine.leaveChannel(); await _engine.release(); } @@ -89,12 +97,30 @@ class _State extends State { await _engine.enableVideo(); - _engine.getMediaEngine().registerAudioFrameObserver(AudioFrameObserver( - onRecordAudioFrame: (channelId, audioFrame) { + _audioFrameObserver = AudioFrameObserver( + onRecordAudioFrame: (channelId, audioFrame) async { debugPrint( '[onRecordAudioFrame] channelId: $channelId, audioFrame: ${audioFrame.toJson()}'); + if (!isJoined) { + return; + } + if (audioFrame.buffer != null) { + await _audioFile.writeAsBytes(audioFrame.buffer!.toList(), + mode: FileMode.append, flush: true); + } }, - )); + onPlaybackAudioFrame: (String channelId, AudioFrame audioFrame) async { + debugPrint( + '[onPlaybackAudioFrame] channelId: $channelId, audioFrame: ${audioFrame.toJson()}'); + if (!isJoined) { + return; + } + if (audioFrame.buffer != null) { + await _playbackAudioFile.writeAsBytes(audioFrame.buffer!.toList(), + mode: FileMode.append, flush: true); + } + }, + ); await _engine.setVideoEncoderConfiguration( const VideoEncoderConfiguration( @@ -106,11 +132,44 @@ class _State extends State { await _engine.startPreview(); + await _startAudioFrameRecord(); + setState(() { _isReadyPreview = true; }); } + Future _startAudioFrameRecord() async { + Directory appDocDir = Platform.isAndroid + ? (await getExternalStorageDirectory())! + : await getApplicationDocumentsDirectory(); + + _audioFile = File(path.join(appDocDir.absolute.path, 'record_audio.raw')); + if (await _audioFile.exists()) { + await _audioFile.delete(); + } + await _audioFile.create(); + logSink + .log('onRecordAudioFrame file output to: ${_audioFile.absolute.path}'); + + _playbackAudioFile = File(path.join( + appDocDir.absolute.path, + 'playback_audio.raw', + )); + if (await _playbackAudioFile.exists()) { + await _playbackAudioFile.delete(); + } + await _playbackAudioFile.create(); + logSink.log( + 'onPlaybackAudioFrame file output to: ${_playbackAudioFile.absolute.path}'); + + _engine.getMediaEngine().registerAudioFrameObserver(_audioFrameObserver); + } + + void _stopAudioFrameRecord() { + _engine.getMediaEngine().unregisterAudioFrameObserver(_audioFrameObserver); + } + Future _joinChannel() async { await _engine.joinChannel( token: config.token, @@ -139,39 +198,8 @@ class _State extends State { return ExampleActionsWidget( displayContentBuilder: (context, isLayoutHorizontal) { if (!_isReadyPreview) return Container(); - return Stack( - children: [ - AgoraVideoView( - controller: VideoViewController( - rtcEngine: _engine, - canvas: const VideoCanvas(uid: 0), - useFlutterTexture: _isUseFlutterTexture, - ), - ), - Align( - alignment: Alignment.topLeft, - child: SingleChildScrollView( - scrollDirection: Axis.horizontal, - child: Row( - children: List.of(remoteUid.map( - (e) => SizedBox( - width: 120, - height: 120, - child: AgoraVideoView( - controller: VideoViewController.remote( - rtcEngine: _engine, - canvas: VideoCanvas(uid: e), - connection: - RtcConnection(channelId: _controller.text), - useFlutterTexture: _isUseFlutterTexture, - ), - ), - ), - )), - ), - ), - ) - ], + return const Center( + child: Text('No Preview'), ); }, actionsBuilder: (context, isLayoutHorizontal) { diff --git a/ios/agora_rtc_engine.podspec b/ios/agora_rtc_engine.podspec index 0485676c4..f31e603e6 100644 --- a/ios/agora_rtc_engine.podspec +++ b/ios/agora_rtc_engine.podspec @@ -17,7 +17,7 @@ Pod::Spec.new do |s| s.source = { :path => '.' } s.source_files = 'Classes/**/*.{h,mm,m,swift}' s.dependency 'Flutter' - s.dependency 'AgoraIrisRTC_iOS', '4.0.0-rc.1' + s.dependency 'AgoraIrisRTC_iOS', '4.0.0-rc.2' s.dependency 'AgoraRtcEngine_Special_iOS', '4.0.0.5' # s.dependency 'AgoraRtcWrapper' s.platform = :ios, '9.0' diff --git a/lib/src/impl/agora_media_engine_impl_override.dart b/lib/src/impl/agora_media_engine_impl_override.dart index d5ebd97b3..b1c29c819 100644 --- a/lib/src/impl/agora_media_engine_impl_override.dart +++ b/lib/src/impl/agora_media_engine_impl_override.dart @@ -5,6 +5,8 @@ import 'package:agora_rtc_engine/agora_rtc_engine.dart'; import 'package:agora_rtc_engine/src/binding/agora_media_base_event_impl.dart'; import 'package:agora_rtc_engine/src/binding/agora_media_engine_impl.dart' as media_engine_impl_binding; +import 'package:agora_rtc_engine/src/binding/call_api_event_handler_buffer_ext.dart'; +import 'package:agora_rtc_engine/src/binding/event_handler_param_json.dart'; import 'package:agora_rtc_engine/src/impl/agora_rtc_engine_impl.dart'; import 'package:agora_rtc_engine/src/impl/api_caller.dart'; import 'package:agora_rtc_engine/src/impl/disposable_object.dart'; @@ -12,6 +14,80 @@ import 'package:iris_event/iris_event.dart'; // ignore_for_file: public_member_api_docs, unused_local_variable +class AudioFrameObserverWrapperOverride extends AudioFrameObserverWrapper { + AudioFrameObserverWrapperOverride(AudioFrameObserver audioFrameObserver) + : super(audioFrameObserver); + + @override + void onEvent(String event, String data, List buffers) { + if (!event.startsWith('AudioFrameObserver')) return; + + final jsonMap = jsonDecode(data); + switch (event) { + case 'AudioFrameObserver_onPlaybackAudioFrameBeforeMixing': + if (audioFrameObserver.onPlaybackAudioFrameBeforeMixing == null) break; + AudioFrameObserverOnPlaybackAudioFrameBeforeMixingJson paramJson = + AudioFrameObserverOnPlaybackAudioFrameBeforeMixingJson.fromJson( + jsonMap); + paramJson = paramJson.fillBuffers(buffers); + String? channelId = paramJson.channelId; + int? uid = paramJson.uid; + AudioFrame? audioFrame = paramJson.audioFrame; + if (channelId == null || uid == null || audioFrame == null) { + break; + } + audioFrame = audioFrame.fillBuffers(buffers); + audioFrameObserver.onPlaybackAudioFrameBeforeMixing!( + channelId, uid, audioFrame); + break; + + case 'AudioFrameObserver_onRecordAudioFrame': + if (audioFrameObserver.onRecordAudioFrame == null) break; + AudioFrameObserverBaseOnRecordAudioFrameJson paramJson = + AudioFrameObserverBaseOnRecordAudioFrameJson.fromJson(jsonMap); + paramJson = paramJson.fillBuffers(buffers); + String? channelId = paramJson.channelId; + AudioFrame? audioFrame = paramJson.audioFrame; + if (channelId == null || audioFrame == null) { + break; + } + audioFrame = audioFrame.fillBuffers(buffers); + audioFrameObserver.onRecordAudioFrame!(channelId, audioFrame); + break; + + case 'AudioFrameObserver_onPlaybackAudioFrame': + if (audioFrameObserver.onPlaybackAudioFrame == null) break; + AudioFrameObserverBaseOnPlaybackAudioFrameJson paramJson = + AudioFrameObserverBaseOnPlaybackAudioFrameJson.fromJson(jsonMap); + paramJson = paramJson.fillBuffers(buffers); + String? channelId = paramJson.channelId; + AudioFrame? audioFrame = paramJson.audioFrame; + if (channelId == null || audioFrame == null) { + break; + } + audioFrame = audioFrame.fillBuffers(buffers); + audioFrameObserver.onPlaybackAudioFrame!(channelId, audioFrame); + break; + + case 'AudioFrameObserver_onMixedAudioFrame': + if (audioFrameObserver.onMixedAudioFrame == null) break; + AudioFrameObserverBaseOnMixedAudioFrameJson paramJson = + AudioFrameObserverBaseOnMixedAudioFrameJson.fromJson(jsonMap); + paramJson = paramJson.fillBuffers(buffers); + String? channelId = paramJson.channelId; + AudioFrame? audioFrame = paramJson.audioFrame; + if (channelId == null || audioFrame == null) { + break; + } + audioFrame = audioFrame.fillBuffers(buffers); + audioFrameObserver.onMixedAudioFrame!(channelId, audioFrame); + break; + default: + break; + } + } +} + class MediaEngineImpl extends media_engine_impl_binding.MediaEngineImpl implements IrisEventHandler, AsyncDisposableObject { MediaEngineImpl._(this._rtcEngine) { @@ -37,7 +113,7 @@ class MediaEngineImpl extends media_engine_impl_binding.MediaEngineImpl unregisterName: 'MediaEngine_unregisterAudioFrameObserver'), jsonEncode(param)); - _eventHandlers.add(AudioFrameObserverWrapper(observer)); + _eventHandlers.add(AudioFrameObserverWrapperOverride(observer)); } @override @@ -77,7 +153,7 @@ class MediaEngineImpl extends media_engine_impl_binding.MediaEngineImpl unregisterName: 'MediaEngine_unregisterAudioFrameObserver'), jsonEncode(param)); - _eventHandlers.remove(AudioFrameObserverWrapper(observer)); + _eventHandlers.remove(AudioFrameObserverWrapperOverride(observer)); } @override diff --git a/lib/src/impl/agora_media_recorder_impl_override.dart b/lib/src/impl/agora_media_recorder_impl_override.dart index cc2f7f25c..40a7e4258 100644 --- a/lib/src/impl/agora_media_recorder_impl_override.dart +++ b/lib/src/impl/agora_media_recorder_impl_override.dart @@ -60,8 +60,6 @@ class MediaRecorderImpl extends media_recorder_impl_binding.MediaRecorderImpl jsonEncode({})); _eventHandlers.clear(); } - - _rtcEngine.removeFromPool(MediaRecorderImpl); } @override diff --git a/lib/src/impl/agora_rtc_engine_impl.dart b/lib/src/impl/agora_rtc_engine_impl.dart index 2f8737750..30a9e2645 100644 --- a/lib/src/impl/agora_rtc_engine_impl.dart +++ b/lib/src/impl/agora_rtc_engine_impl.dart @@ -66,8 +66,9 @@ class ObjectPool { } Future clear() async { - for (final key in pool.keys) { - await pool[key]?.disposeAsync(); + final values = pool.values; + for (final v in values) { + await v.disposeAsync(); } pool.clear(); @@ -244,7 +245,9 @@ class RtcEngineImpl extends rtc_engine_ex_binding.RtcEngineExImpl release(sync: true); }, ); - WidgetsBinding.instance.addObserver(_lifecycle!); + // Compatible with 2.10 + // ignore: invalid_null_aware_operator + WidgetsBinding.instance?.addObserver(_lifecycle!); if (defaultTargetPlatform == TargetPlatform.android) { final externalFilesDir = @@ -282,7 +285,9 @@ class RtcEngineImpl extends rtc_engine_ex_binding.RtcEngineExImpl if (_instance == null) return; if (_lifecycle != null) { - WidgetsBinding.instance.removeObserver(_lifecycle!); + // Compatible with 2.10 + // ignore: invalid_null_aware_operator + WidgetsBinding.instance?.removeObserver(_lifecycle!); _lifecycle = null; } @@ -1084,4 +1089,9 @@ class VideoDeviceManagerImpl extends rtc_engine_binding.VideoDeviceManagerImpl Future release() async { _instance = null; } + + @override + Future startDeviceTest(int hwnd) { + throw AgoraRtcException(code: ErrorCodeType.errNotSupported.value()); + } } diff --git a/lib/src/impl/agora_spatial_audio_impl_override.dart b/lib/src/impl/agora_spatial_audio_impl_override.dart index b6d5f662c..401d4dc33 100644 --- a/lib/src/impl/agora_spatial_audio_impl_override.dart +++ b/lib/src/impl/agora_spatial_audio_impl_override.dart @@ -47,8 +47,6 @@ class LocalSpatialAudioEngineImpl extends spatial_audio_binding final param = createParams({}); final callApiResult = await apiCaller.callIrisApi(apiType, jsonEncode(param), buffers: null); - - _rtcEngine.removeFromPool(LocalSpatialAudioEngineImpl); } @override diff --git a/lib/src/impl/api_caller.dart b/lib/src/impl/api_caller.dart index f2ab43269..ad0a5ecec 100644 --- a/lib/src/impl/api_caller.dart +++ b/lib/src/impl/api_caller.dart @@ -635,6 +635,7 @@ class _ApiCallExecutorInternal implements _ApiCallExecutorBase { )); } else if (key.op == CallIrisEventOp.dispose) { _irisEventHandlerObservers[key]?.dispose(); + _irisEventHandlerObservers.remove(key); } } else if (key is IrisEventObserverKey) { if (key.op == CallIrisEventOp.create) { @@ -650,6 +651,7 @@ class _ApiCallExecutorInternal implements _ApiCallExecutorBase { )); } else if (key.op == CallIrisEventOp.dispose) { _irisEventHandlerObservers[key]?.dispose(); + _irisEventHandlerObservers.remove(key); } } diff --git a/macos/agora_rtc_engine.podspec b/macos/agora_rtc_engine.podspec index c67db93aa..19c499452 100644 --- a/macos/agora_rtc_engine.podspec +++ b/macos/agora_rtc_engine.podspec @@ -16,8 +16,8 @@ A new flutter plugin project. s.source_files = 'Classes/**/*.{h,mm}', 'Classes/File.swift' s.dependency 'FlutterMacOS' # s.dependency 'AgoraRtcWrapper' - s.dependency 'AgoraRtcEngine_Special_macOS', '4.0.0.5' - s.dependency 'AgoraIrisRTC_macOS', '4.0.0-rc.1' + s.dependency 'AgoraRtcEngine_Special_macOS', '4.0.0.5' + s.dependency 'AgoraIrisRTC_macOS', '4.0.0-rc.2' s.platform = :osx, '10.11' s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' } diff --git a/pubspec.yaml b/pubspec.yaml index befa1ce49..e1bd6da12 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -21,7 +21,7 @@ dependencies: dev_dependencies: flutter_test: sdk: flutter - flutter_lints: ^2.0.0 + flutter_lints: ^1.0.0 build_runner: ^2.1.7 json_serializable: ^6.1.1 ffigen: '>=4.1.2' diff --git a/scripts/flutter-build-runner.sh b/scripts/flutter-build-runner.sh index 938ce2cbd..cbc79b209 100644 --- a/scripts/flutter-build-runner.sh +++ b/scripts/flutter-build-runner.sh @@ -8,4 +8,4 @@ rm -rf $AGORA_FLUTTER_PROJECT_PATH/example/macos/Flutter/ephemeral rm -rf $AGORA_FLUTTER_PROJECT_PATH/example/windows/Flutter/ephemeral rm -rf $AGORA_FLUTTER_PROJECT_PATH/example/ios/.symlinks -# flutter packages pub run build_runner build --delete-conflicting-outputs \ No newline at end of file +flutter packages pub run build_runner build --delete-conflicting-outputs \ No newline at end of file diff --git a/windows/CMakeLists.txt b/windows/CMakeLists.txt index 2ba23fdc8..f83e92158 100644 --- a/windows/CMakeLists.txt +++ b/windows/CMakeLists.txt @@ -12,8 +12,8 @@ project(${PROJECT_NAME} LANGUAGES CXX) # not be changed set(PLUGIN_NAME "agora_rtc_engine_plugin") -set(IRIS_SDK_DOWNLOAD_URL "https://download.agora.io/sdk/release/iris_4.0.0_DCG_Windows_Video_20220910_0326.zip") -set(IRIS_SDK_DOWNLOAD_NAME "iris_4.0.0_DCG_Windows_20220910_0326") +set(IRIS_SDK_DOWNLOAD_URL "https://download.agora.io/sdk/release/iris_4.0.0_DCG_Windows_Video_20220915_0852.zip") +set(IRIS_SDK_DOWNLOAD_NAME "iris_4.0.0_DCG_Windows_20220915_0852") set(RTC_SDK_DOWNLOAD_NAME "Agora_Native_SDK_for_Windows_FULL") set(IRIS_SDK_VERSION "v3_6_2_fix.1")