From f09a37ddf9fbdd137782ec00463e5b8a7bc6fe9a Mon Sep 17 00:00:00 2001 From: jpericas22 Date: Tue, 31 Jul 2018 17:34:56 +0200 Subject: [PATCH] Fixes for html5 audio and video with emscripten (#6039) SoundPlayer.isLoaded() was giving a wrong output on html5 audio. It returned if the loading started, not if the loading was finished. This is caused by the way the audio is fetched (asynchronous). Video grabbing from webcam was outdated and did not work on IOS11 and Safari 11.x. This fixes problems, updates the deprecated navigator.getUserMedia() to navigator.mediaDevices.getUserMedia and includes a fix to get webcam support on both IOS and Safari. --- .../libs/html5audio/include/html5audio.h | 1 + .../lib/emscripten/library_html5audio.js | 73 +++++----- .../lib/emscripten/library_html5video.js | 129 +++++++++--------- .../src/ofxEmscriptenSoundPlayer.cpp | 2 +- 4 files changed, 104 insertions(+), 101 deletions(-) diff --git a/addons/ofxEmscripten/libs/html5audio/include/html5audio.h b/addons/ofxEmscripten/libs/html5audio/include/html5audio.h index 76627239cfe..6e72a4afb67 100644 --- a/addons/ofxEmscripten/libs/html5audio/include/html5audio.h +++ b/addons/ofxEmscripten/libs/html5audio/include/html5audio.h @@ -23,4 +23,5 @@ extern "C"{ extern int html5audio_stream_create(int context_id, int bufferSize, int inputChannels, int outputChannels, float * inbuffer, float * outbuffer, html5audio_stream_callback callback, void * userData); extern int html5audio_stream_free(int stream); + extern bool html5audio_sound_is_loaded(int sound); } diff --git a/addons/ofxEmscripten/libs/html5audio/lib/emscripten/library_html5audio.js b/addons/ofxEmscripten/libs/html5audio/lib/emscripten/library_html5audio.js index 1e22a632bbf..8b971c522fd 100644 --- a/addons/ofxEmscripten/libs/html5audio/lib/emscripten/library_html5audio.js +++ b/addons/ofxEmscripten/libs/html5audio/lib/emscripten/library_html5audio.js @@ -3,17 +3,17 @@ var LibraryHTML5Audio = { contexts: [], ffts: [], lastContextID: 0, - + soundBuffers: [], soundSources: [], soundStartTimes: [], soundGains: [], lastSoundID: 0, - + streams: [], mediaElements: [], lastStreamID: 0, - + soundPosition: function(sound_id){ var source = AUDIO.soundSources[sound_id]; if(source!=undefined){ @@ -23,10 +23,10 @@ var LibraryHTML5Audio = { return Math.min(duration,playTime); }else{ return 0; - } + } } }, - + html5audio_context_create: function(){ try { // Fix up for prefixing @@ -46,32 +46,32 @@ var LibraryHTML5Audio = { return -1; } }, - + html5audio_context_spectrum: function(context_id, bands, spectrum){ AUDIO.ffts[context_id].fftSize = bands*2; var spectrumArray = Module.HEAPF32.subarray(spectrum>>2, (spectrum>>2)+bands); AUDIO.ffts[context_id].getFloatFrequencyData(spectrumArray); }, - + html5audio_context_samplerate: function(context){ return AUDIO.contexts[context_id].sampleRate.value; }, - + html5audio_sound_load: function(context_id, url){ var request = new XMLHttpRequest(); request.open('GET', Pointer_stringify(url), true); request.responseType = 'arraybuffer'; - + var id = AUDIO.lastSoundID++; AUDIO.soundGains[id] = AUDIO.contexts[context_id].createGain(); AUDIO.soundGains[id].connect(AUDIO.ffts[context_id]); - + // Decode asynchronously request.onload = function() { - AUDIO.contexts[context_id].decodeAudioData(request.response, + AUDIO.contexts[context_id].decodeAudioData(request.response, function(buffer) { AUDIO.soundBuffers[id] = buffer; - }, + }, function(e){ console.log("couldn't decode sound " + id, e); } @@ -80,7 +80,7 @@ var LibraryHTML5Audio = { request.send(); return id; }, - + html5audio_sound_play: function(context_id, sound_id, offset){ if(AUDIO.soundBuffers[sound_id]!=undefined){ if(AUDIO.contexts[context_id]!=undefined && AUDIO.contexts[context_id].paused){ @@ -95,7 +95,7 @@ var LibraryHTML5Audio = { source.paused = false; source.onended = function(event){ event.target.done = true; - } + } AUDIO.soundSources[sound_id] = source; source.startTime = AUDIO.contexts[context_id].currentTime - offset; source.start(offset); @@ -111,13 +111,13 @@ var LibraryHTML5Audio = { AUDIO.soundSources[sound_id].stop(); AUDIO.soundSources[sound_id].paused = true; }, - + html5audio_sound_rate: function(sound_id){ if(AUDIO.soundSources[sound_id]!=undefined){ return AUDIO.soundSources[sound_id].playbackRate.value; } }, - + html5audio_sound_set_rate: function(sound_id,rate){ var source = AUDIO.soundSources[sound_id]; if(source!=undefined){ @@ -126,7 +126,7 @@ var LibraryHTML5Audio = { AUDIO.soundSources[sound_id].playbackRate.value = rate; } }, - + html5audio_sound_done: function(sound_id){ if(AUDIO.soundSources[sound_id]!=undefined){ return AUDIO.soundSources[sound_id].done; @@ -134,7 +134,7 @@ var LibraryHTML5Audio = { return false; } }, - + html5audio_sound_duration: function(sound_id){ if(AUDIO.soundBuffers[sound_id]!=undefined){ return AUDIO.soundBuffers[sound_id].duration; @@ -146,33 +146,33 @@ var LibraryHTML5Audio = { html5audio_sound_position: function(sound_id){ return AUDIO.soundPosition(sound_id); }, - + html5audio_sound_set_loop: function(sound_id, loop){ AUDIO.soundSources[sound_id].loop = loop; }, - + html5audio_sound_set_gain: function(sound_id, gain){ AUDIO.soundGains[sound_id].gain = gain; }, - + html5audio_sound_gain: function(sound_id){ return AUDIO.soundGains[sound_id].gain; }, - + html5audio_sound_free: function(sound_id){ return AUDIO.soundBuffers[sound_id] = null; return AUDIO.soundSources[sound_id] = null; return AUDIO.soundStartTimes[sound_id] = 0; return AUDIO.soundGains[sound_id] = null; }, - + html5audio_stream_create: function(context_id, bufferSize, inputChannels, outputChannels, inbuffer, outbuffer, callback, userData){ var stream = AUDIO.contexts[context_id].createScriptProcessor(bufferSize,inputChannels,outputChannels); var inbufferArray = Module.HEAPF32.subarray(inbuffer>>2,(inbuffer>>2)+bufferSize*inputChannels); var outbufferArray = Module.HEAPF32.subarray(outbuffer>>2,(outbuffer>>2)+bufferSize*outputChannels); - + var id = AUDIO.lastStreamID++; - + stream.onaudioprocess = function(event){ var i,j,c; if(inputChannels>0){ @@ -183,9 +183,9 @@ var LibraryHTML5Audio = { } } } - + Runtime.dynCall('viiii',callback, [bufferSize,inputChannels,outputChannels,userData]); - + if(outputChannels>0){ for(c=0;c0){ navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.msGetUserMedia; - + if(navigator.getUserMedia){ navigator.getUserMedia( - {audio: true}, + {audio: true}, function(audioIn) { var mediaElement = AUDIO.contexts[context_id].createMediaStreamSource(audioIn); mediaElement.connect(stream); @@ -216,18 +216,25 @@ var LibraryHTML5Audio = { ); } } - + stream.connect(AUDIO.ffts[context_id]); AUDIO.streams[id] = stream; return id; }, - + html5audio_stream_free: function(stream_id){ return AUDIO.streams[stream_id] = null; return AUDIO.mediaElements[stream_id] = null; + }, + + html5audio_sound_is_loaded: function(sound){ + if(sound!=-1 && AUDIO.soundBuffers[sound] != undefined){ + return true; + } + return false; } } autoAddDeps(LibraryHTML5Audio, '$AUDIO'); -mergeInto(LibraryManager.library, LibraryHTML5Audio); \ No newline at end of file +mergeInto(LibraryManager.library, LibraryHTML5Audio); diff --git a/addons/ofxEmscripten/libs/html5video/lib/emscripten/library_html5video.js b/addons/ofxEmscripten/libs/html5video/lib/emscripten/library_html5video.js index 564b40ae761..21fca0b5d43 100644 --- a/addons/ofxEmscripten/libs/html5video/lib/emscripten/library_html5video.js +++ b/addons/ofxEmscripten/libs/html5video/lib/emscripten/library_html5video.js @@ -3,28 +3,28 @@ var LibraryHTML5Video = { players: [], playersContexts: [], playersCounter: 0, - + getNewPlayerId: function() { var ret = VIDEO.playersCounter++; return ret; }, - + grabbers: [], grabbersContexts: [], grabbersCounter: 0, - + getNewGrabberId: function() { var ret = VIDEO.grabbersCounter++; return ret; }, - + getUserMedia: function(){ return navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.msGetUserMedia; }, - + update: function(updatePixels, video, context, dstPixels){ if((updatePixels || video.pixelFormat!="RGBA") && video.width!=0 && video.height!=0 && dstPixels!=0){ try { @@ -55,7 +55,7 @@ var LibraryHTML5Video = { array[i++] = (((srcPixels[j++]|0) << 1) + ((srcPixels[j]|0) << 2) + (srcPixels[j++]|0) + (srcPixels[j++]|0)) >> 3; ++j; } - + GLctx.bindTexture(GLctx.TEXTURE_2D, GL.textures[video.textureId]); GLctx.texImage2D(GLctx.TEXTURE_2D, 0, GLctx.LUMINANCE, video.width, video.height, 0, GLctx.LUMINANCE, GLctx.UNSIGNED_BYTE, array); GLctx.bindTexture(GLctx.TEXTURE_2D, null); @@ -69,20 +69,20 @@ var LibraryHTML5Video = { } } }, - + html5video_player_create: function(){ var video = document.createElement('video'); video.loop = true; video.pixelFormat = "RGB"; //video.crossOrigin = 'anonymous'; - + var player_id = VIDEO.getNewPlayerId(); VIDEO.players[player_id] = video; video.onloadedmetadata = function (e){ console.log(this.videoWidth + 'x' + this.videoHeight); VIDEO.players[player_id].width = this.videoWidth; VIDEO.players[player_id].height = this.videoHeight; - + var videoImage = document.createElement( 'canvas' ); videoImage.width = this.videoWidth; videoImage.height = this.videoHeight; @@ -91,13 +91,13 @@ var LibraryHTML5Video = { // background color if no video present videoImageContext.fillStyle = '#000000'; videoImageContext.fillRect( 0, 0, videoImage.width, videoImage.height ); - + VIDEO.playersContexts[player_id] = videoImageContext; }; - + return player_id; }, - + html5video_player_delete: function(id){ VIDEO.players[id] = null; }, @@ -118,15 +118,15 @@ var LibraryHTML5Video = { VIDEO.players[id].textureId = texId; VIDEO.players[id].load(); }, - + html5video_player_pixel_format: function(id){ return allocate(intArrayFromString(VIDEO.players[id].pixelFormat), 'i8', ALLOC_STACK); }, - + html5video_player_set_pixel_format: function(id, format){ VIDEO.players[id].pixelFormat = Pointer_stringify(format); }, - + html5video_player_update__deps: ['$GL'], html5video_player_update: function(id,update_pixels,pixels){ var player = VIDEO.players[id]; @@ -140,93 +140,90 @@ var LibraryHTML5Video = { return false; } }, - + html5video_player_texture_id: function(id){ return VIDEO.players[id].textureId; }, - + html5video_player_width: function(id){ return VIDEO.players[id].width; }, - + html5video_player_height: function(id){ return VIDEO.players[id].height; }, - + html5video_player_play: function(id){ console.log('play'); VIDEO.players[id].play(); }, - + html5video_player_pause: function(id){ VIDEO.players[id].pause(); }, - + html5video_player_stop: function(id){ VIDEO.players[id].pause(); }, - + html5video_player_is_paused: function(id){ return VIDEO.players[id].paused; }, - + html5video_player_ready_state: function(id){ return VIDEO.players[id].readyState; }, - + html5video_player_duration: function(id){ return VIDEO.players[id].duration; }, - + html5video_player_current_time: function(id){ return VIDEO.players[id].currentTime; }, - + html5video_player_set_current_time: function(id,time){ VIDEO.players[id].currentTime = time; }, - + html5video_player_ended: function(id){ return VIDEO.players[id].ended; }, - + html5video_player_playback_rate: function(id){ return VIDEO.players[id].playbackRate; }, - + html5video_player_set_playback_rate: function(id,rate){ VIDEO.players[id].playbackRate = rate; }, - + html5video_player_volume: function(id){ return VIDEO.players[id].volume; }, - + html5video_player_set_volume: function(id,volume){ VIDEO.players[id].volume = volume; }, - + html5video_player_set_loop: function(id,loop){ VIDEO.players[id].loop = loop; }, - + html5video_player_loop: function(id){ return VIDEO.players[id].loop; }, - + html5video_grabber_create: function(){ - if( VIDEO.getUserMedia() ){ + var video = document.createElement('video'); video.autoplay=true; video.pixelFormat = "RGB"; - + var grabber_id = VIDEO.getNewGrabberId(); VIDEO.grabbers[grabber_id] = video; return grabber_id; - }else{ - console.log("coudln't create grabber"); - return -1; - } + }, html5video_grabber_init__deps: ['$GL'], @@ -234,7 +231,7 @@ var LibraryHTML5Video = { if(id!=-1){ VIDEO.grabbers[id].width = w; VIDEO.grabbers[id].height = h; - + var videoImage = document.createElement( 'canvas' ); videoImage.width = w; videoImage.height = h; @@ -243,9 +240,9 @@ var LibraryHTML5Video = { // background color if no video present videoImageContext.fillStyle = '#000000'; videoImageContext.fillRect( 0, 0, w, h ); - + VIDEO.grabbersContexts[id] = videoImageContext; - + var errorCallback = function(e) { console.log('Couldn\'t init grabber!', e); }; @@ -266,38 +263,36 @@ var LibraryHTML5Video = { maxWidth: w, maxHeight: h, }, - optional: [ + optional: [ { minFrameRate: framerate } ] } }; } - var getUserMedia = VIDEO.getUserMedia().bind(navigator); - getUserMedia(constraints, function(stream) { - VIDEO.grabbers[id].src = window.URL.createObjectURL(stream); - var texId = GL.getNewId(GL.textures); - var texture = GLctx.createTexture(); - texture.name = texId; - GL.textures[texId] = texture; - GLctx.bindTexture(GLctx.TEXTURE_2D, texture); - GLctx.texParameteri(GLctx.TEXTURE_2D, GLctx.TEXTURE_MAG_FILTER, GLctx.LINEAR); - GLctx.texParameteri(GLctx.TEXTURE_2D, GLctx.TEXTURE_MIN_FILTER, GLctx.LINEAR); - GLctx.texParameteri(GLctx.TEXTURE_2D, GLctx.TEXTURE_WRAP_S, GLctx.CLAMP_TO_EDGE); - GLctx.texParameteri(GLctx.TEXTURE_2D, GLctx.TEXTURE_WRAP_T, GLctx.CLAMP_TO_EDGE); - VIDEO.grabbers[id].textureId = texId; - }, errorCallback); + navigator.mediaDevices.getUserMedia(constraints) + .then(function(stream) { + window.stream = stream; + VIDEO.grabbers[id].srcObject = stream + VIDEO.grabbers[id].onloadedmetadata = function (e){ + VIDEO.grabbers[id].play(); + } + }) + .catch(function(err) { + console.log(e); + }); + } }, - + html5video_grabber_pixel_format: function(id){ return allocate(intArrayFromString(VIDEO.grabbers[id].pixelFormat), 'i8', ALLOC_STACK); }, - + html5video_grabber_set_pixel_format: function(id, format){ VIDEO.grabbers[id].pixelFormat = Pointer_stringify(format); }, - + html5video_grabber_update__deps: ['$GL'], html5video_grabber_update: function(id,update_pixels,pixels){ var grabber = VIDEO.grabbers[id]; @@ -308,24 +303,24 @@ var LibraryHTML5Video = { return false; } }, - + html5video_grabber_texture_id: function(id){ return VIDEO.grabbers[id].textureId; }, - + html5video_grabber_width: function(id){ return VIDEO.grabbers[id].width; }, - + html5video_grabber_height: function(id){ return VIDEO.grabbers[id].height; }, - + html5video_grabber_ready_state: function(id){ return VIDEO.grabbers[id].readyState; }, - - + + } diff --git a/addons/ofxEmscripten/src/ofxEmscriptenSoundPlayer.cpp b/addons/ofxEmscripten/src/ofxEmscriptenSoundPlayer.cpp index bc977439578..d1a203c2970 100644 --- a/addons/ofxEmscripten/src/ofxEmscriptenSoundPlayer.cpp +++ b/addons/ofxEmscripten/src/ofxEmscriptenSoundPlayer.cpp @@ -162,7 +162,7 @@ float ofxEmscriptenSoundPlayer::getPan() const{ } bool ofxEmscriptenSoundPlayer::isLoaded() const{ - return sound!=-1; + return html5audio_sound_is_loaded(sound); } float ofxEmscriptenSoundPlayer::getVolume() const{