Add in the microphone access to the Input object

Author: goatchurchprime

core/input/input.cpp

@@ -35,6 +35,7 @@
 #include "core/input/default_controller_mappings.h"
 #include "core/input/input_map.h"
 #include "core/os/os.h"
+#include "servers/audio_server.h"
 
 #ifdef DEV_ENABLED
 #include "core/os/thread.h"
@@ -170,6 +171,10 @@ void Input::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("is_emulating_mouse_from_touch"), &Input::is_emulating_mouse_from_touch);
 	ClassDB::bind_method(D_METHOD("set_emulate_touch_from_mouse", "enable"), &Input::set_emulate_touch_from_mouse);
 	ClassDB::bind_method(D_METHOD("is_emulating_touch_from_mouse"), &Input::is_emulating_touch_from_mouse);
+	ClassDB::bind_method(D_METHOD("start_microphone"), &Input::start_microphone);
+	ClassDB::bind_method(D_METHOD("stop_microphone"), &Input::stop_microphone);
+	ClassDB::bind_method(D_METHOD("get_microphone_frames_available"), &Input::get_microphone_frames_available);
+	ClassDB::bind_method(D_METHOD("get_microphone_buffer", "frames"), &Input::get_microphone_buffer);
 
 	ADD_PROPERTY(PropertyInfo(Variant::INT, "mouse_mode"), "set_mouse_mode", "get_mouse_mode");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "use_accumulated_input"), "set_use_accumulated_input", "is_using_accumulated_input");
@@ -1837,6 +1842,53 @@ bool Input::is_input_disabled() const {
 	return disable_input;
 }
 
+Error Input::start_microphone() {
+	if (!GLOBAL_GET("audio/driver/enable_input")) {
+		WARN_PRINT("You must enable the project setting \"audio/driver/enable_input\" to use audio capture.");
+		return FAILED;
+	}
+
+	microphone_buffer_ofs = 0;
+	return AudioDriver::get_singleton()->input_start();
+}
+
+Error Input::stop_microphone() {
+	return AudioDriver::get_singleton()->input_stop();
+}
+
+int Input::get_microphone_frames_available() {
+	unsigned int input_position = AudioDriver::get_singleton()->get_input_position();
+	if (input_position < microphone_buffer_ofs) {
+		Vector<int32_t> &buf = AudioDriver::get_singleton()->get_input_buffer();
+		input_position += buf.size();
+	}
+	return (input_position - microphone_buffer_ofs) / 2;
+}
+
+PackedVector2Array Input::get_microphone_buffer(int p_frames) {
+	PackedVector2Array ret;
+	unsigned int input_position = AudioDriver::get_singleton()->get_input_position();
+	Vector<int32_t> &buf = AudioDriver::get_singleton()->get_input_buffer();
+	if (input_position < microphone_buffer_ofs) {
+		input_position += buf.size();
+	}
+	if ((microphone_buffer_ofs + p_frames * 2 <= input_position) && (p_frames >= 0)) {
+		ret.resize(p_frames);
+		for (int i = 0; i < p_frames; i++) {
+			float l = (buf[microphone_buffer_ofs++] >> 16) / 32768.f;
+			if (microphone_buffer_ofs >= buf.size()) {
+				microphone_buffer_ofs = 0;
+			}
+			float r = (buf[microphone_buffer_ofs++] >> 16) / 32768.f;
+			if (microphone_buffer_ofs >= buf.size()) {
+				microphone_buffer_ofs = 0;
+			}
+			ret.write[i] = Vector2(l, r);
+		}
+	}
+	return ret;
+}
+
 Input::Input() {
 	singleton = this;
 

core/input/input.h

@@ -105,6 +105,7 @@ class Input : public Object {
 	int64_t mouse_window = 0;
 	bool legacy_just_pressed_behavior = false;
 	bool disable_input = false;
+	unsigned int microphone_buffer_ofs = 0;
 
 	struct ActionState {
 		uint64_t pressed_physics_frame = UINT64_MAX;
@@ -396,6 +397,11 @@ class Input : public Object {
 	void set_disable_input(bool p_disable);
 	bool is_input_disabled() const;
 
+	Error start_microphone();
+	Error stop_microphone();
+	int get_microphone_frames_available();
+	PackedVector2Array get_microphone_buffer(int p_frames);
+
 	Input();
 	~Input();
 };

doc/classes/Input.xml

@@ -183,6 +183,22 @@
 				[b]Note:[/b] For Android, [member ProjectSettings.input_devices/sensors/enable_magnetometer] must be enabled.
 			</description>
 		</method>
+		<method name="get_microphone_buffer">
+			<return type="PackedVector2Array" />
+			<param index="0" name="frames" type="int" />
+			<description>
+				Gets the next [param frames] audio samples from the internal microphone ring buffer.
+				The buffer is filled at the rate of [method AudioServer.get_input_mix_rate] frames per second after [method start_microphone] has been called.
+				Returns a [PackedVector2Array] containing exactly [param frames] audio samples if available, or an empty [PackedVector2Array] if insufficient data was available.
+				The samples are signed floating-point PCM between [code]-1[/code] and [code]1[/code]. You will have to scale them if you want to use them as 8 or 16-bit integer samples. ([code]v = 0x7fff * samples[0].x[/code])
+			</description>
+		</method>
+		<method name="get_microphone_frames_available">
+			<return type="int" />
+			<description>
+				Returns the number of frames available to read using [method get_microphone_buffer].
+			</description>
+		</method>
 		<method name="get_mouse_button_mask" qualifiers="const">
 			<return type="int" enum="MouseButtonMask" is_bitfield="true" />
 			<description>
@@ -395,13 +411,26 @@
 				[b]Note:[/b] For macOS, vibration is only supported in macOS 11 and later.
 			</description>
 		</method>
+		<method name="start_microphone">
+			<return type="int" enum="Error" />
+			<description>
+				Starts the input microphone.  Call [method get_microphone_buffer] to retrieve the values.
+				Returns [code]ERR_ALREADY_IN_USE[/code] if already running.
+			</description>
+		</method>
 		<method name="stop_joy_vibration">
 			<return type="void" />
 			<param index="0" name="device" type="int" />
 			<description>
 				Stops the vibration of the joypad started with [method start_joy_vibration].
 			</description>
 		</method>
+		<method name="stop_microphone">
+			<return type="int" enum="Error" />
+			<description>
+				Stops the input microphone.
+			</description>
+		</method>
 		<method name="vibrate_handheld">
 			<return type="void" />
 			<param index="0" name="duration_ms" type="int" default="500" />

drivers/pulseaudio/audio_driver_pulseaudio.cpp

@@ -763,6 +763,9 @@ void AudioDriverPulseAudio::finish_input_device() {
 }
 
 Error AudioDriverPulseAudio::input_start() {
+	if (pa_rec_str) {
+		return ERR_ALREADY_IN_USE;
+	}
 	lock();
 	Error err = init_input_device();
 	unlock();

drivers/wasapi/audio_driver_wasapi.cpp

@@ -1004,6 +1004,10 @@ void AudioDriverWASAPI::finish() {
 }
 
 Error AudioDriverWASAPI::input_start() {
+	if (audio_input.active.is_set()) {
+		return ERR_ALREADY_IN_USE;
+	}
+
 	Error err = init_input_device();
 	if (err != OK) {
 		ERR_PRINT("WASAPI: init_input_device error");
@@ -1023,11 +1027,9 @@ Error AudioDriverWASAPI::input_stop() {
 	if (audio_input.active.is_set()) {
 		audio_input.audio_client->Stop();
 		audio_input.active.clear();
-
-		return OK;
 	}
 
-	return FAILED;
+	return OK;
 }
 
 PackedStringArray AudioDriverWASAPI::get_input_device_list() {

platform/android/audio_driver_opensl.cpp

@@ -273,28 +273,30 @@ Error AudioDriverOpenSL::input_start() {
 	}
 
 	if (OS::get_singleton()->request_permission("RECORD_AUDIO")) {
-		return init_input_device();
+		Error err = init_input_device();
+		if (err != OK) {
+			input_stop();
+		}
+		return err;
 	}
 
 	WARN_PRINT("Unable to start audio capture - No RECORD_AUDIO permission");
 	return ERR_UNAUTHORIZED;
 }
 
 Error AudioDriverOpenSL::input_stop() {
-	if (!recordItf || !recordBufferQueueItf) {
-		return ERR_CANT_OPEN;
+	if (recordItf) {
+		(*recordItf)->SetRecordState(recordItf, SL_RECORDSTATE_STOPPED);
+		recordItf = nullptr;
 	}
 
-	SLuint32 state;
-	SLresult res = (*recordItf)->GetRecordState(recordItf, &state);
-	ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN);
-
-	if (state != SL_RECORDSTATE_STOPPED) {
-		res = (*recordItf)->SetRecordState(recordItf, SL_RECORDSTATE_STOPPED);
-		ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN);
-
-		res = (*recordBufferQueueItf)->Clear(recordBufferQueueItf);
-		ERR_FAIL_COND_V(res != SL_RESULT_SUCCESS, ERR_CANT_OPEN);
+	if (recordBufferQueueItf) {
+		(*recordBufferQueueItf)->Clear(recordBufferQueueItf);
+		recordBufferQueueItf = nullptr;
+	}
+	if (recorder) {
+		(*recorder)->Destroy(recorder);
+		recorder = nullptr;
 	}
 
 	return OK;

platform/web/audio_driver_web.cpp

@@ -190,12 +190,16 @@ void AudioDriverWeb::finish() {
 }
 
 Error AudioDriverWeb::input_start() {
+	if (input_started) {
+		return ERR_ALREADY_IN_USE;
+	}
 	lock();
 	input_buffer_init(buffer_length);
 	unlock();
 	if (godot_audio_input_start()) {
 		return FAILED;
 	}
+	input_started = true;
 	return OK;
 }
 
@@ -204,6 +208,7 @@ Error AudioDriverWeb::input_stop() {
 	lock();
 	input_buffer.clear();
 	unlock();
+	input_started = false;
 	return OK;
 }
 

platform/web/audio_driver_web.h

@@ -54,6 +54,7 @@ class AudioDriverWeb : public AudioDriver {
 	int buffer_length = 0;
 	int mix_rate = 0;
 	int channel_count = 0;
+	bool input_started = false;
 
 	WASM_EXPORT static void _state_change_callback(int p_state);
 	WASM_EXPORT static void _latency_update_callback(float p_latency);

servers/audio/audio_stream.cpp

@@ -389,7 +389,7 @@ bool AudioStreamMicrophone::is_monophonic() const {
 int AudioStreamPlaybackMicrophone::_mix_internal(AudioFrame *p_buffer, int p_frames) {
 	AudioDriver::get_singleton()->lock();
 
-	Vector<int32_t> buf = AudioDriver::get_singleton()->get_input_buffer();
+	Vector<int32_t> &buf = AudioDriver::get_singleton()->get_input_buffer();
 	unsigned int input_size = AudioDriver::get_singleton()->get_input_size();
 	int mix_rate = AudioDriver::get_singleton()->get_input_mix_rate();
 	unsigned int playback_delay = MIN(((50 * mix_rate) / 1000) * 2, buf.size() >> 1);

servers/audio_server.cpp

@@ -111,6 +111,7 @@ void AudioDriver::input_buffer_write(int32_t sample) {
 			input_size++;
 		}
 	} else {
+		// This case could only happen if two threads were calling this function without any locking.
 		WARN_PRINT("input_buffer_write: Invalid input_position=" + itos(input_position) + " input_buffer.size()=" + itos(input_buffer.size()));
 	}
 }