From ca0c090c1a98aa37bfaf6d85e1ae681f6d6f5236 Mon Sep 17 00:00:00 2001 From: Drew Hill Date: Sat, 23 Dec 2017 13:25:45 -0500 Subject: [PATCH 1/3] add support in mediacodecaudiorenderer for 24bit pcm to float --- .../audio/FloatResamplingAudioProcessor.java | 171 ++++++++++++++++++ .../audio/MediaCodecAudioRenderer.java | 108 +++++++++-- 2 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java new file mode 100644 index 00000000000..28d2eca25f9 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java @@ -0,0 +1,171 @@ +package com.google.android.exoplayer2.audio; + + +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * An {@link AudioProcessor} that converts audio data to {@link C#ENCODING_PCM_16BIT}. + */ +/* package */ final class FloatResamplingAudioProcessor implements AudioProcessor { + + private int sampleRateHz; + private static final double PCM_INT32_FLOAT = 1.0 / 0x7fffffff; + + private int channelCount; + @C.PcmEncoding + private int sourceEncoding; + private ByteBuffer buffer; + private ByteBuffer outputBuffer; + private boolean inputEnded; + + /** + * Creates a new audio processor that converts audio data to {@link C#ENCODING_PCM_16BIT}. + */ + public FloatResamplingAudioProcessor() { + sampleRateHz = Format.NO_VALUE; + channelCount = Format.NO_VALUE; + sourceEncoding = C.ENCODING_INVALID; + buffer = EMPTY_BUFFER; + outputBuffer = EMPTY_BUFFER; + } + + @Override + public boolean configure(int sampleRateHz, int channelCount, @C.Encoding int encoding) + throws AudioProcessor.UnhandledFormatException { + if (encoding != C.ENCODING_PCM_24BIT) { + throw new AudioProcessor.UnhandledFormatException(sampleRateHz, channelCount, encoding); + } + if (this.sampleRateHz == sampleRateHz && this.channelCount == channelCount + && this.sourceEncoding == encoding) { + return false; + } + this.sampleRateHz = sampleRateHz; + this.channelCount = channelCount; + this.sourceEncoding = encoding; + + return true; + } + + @Override + public boolean isActive() { return sourceEncoding == C.ENCODING_PCM_24BIT; } + + @Override + public int getOutputChannelCount() { return channelCount; } + + @Override + public int getOutputEncoding() { return C.ENCODING_PCM_FLOAT; } + + @Override + public int getOutputSampleRateHz() { + return sampleRateHz; + } + + @Override + public void queueInput(ByteBuffer inputBuffer) { + int offset = inputBuffer.position(); + int limit = inputBuffer.limit(); + int size = limit - offset; + + int resampledSize; + switch (sourceEncoding) { + case C.ENCODING_PCM_24BIT: + resampledSize = (size / 3) * 4; + break; + case C.ENCODING_PCM_32BIT: + case C.ENCODING_PCM_8BIT: + case C.ENCODING_PCM_16BIT: + case C.ENCODING_PCM_FLOAT: + case C.ENCODING_INVALID: + case Format.NO_VALUE: + default: + // Never happens. + throw new IllegalStateException(); + } + + if (buffer.capacity() < resampledSize) { + buffer = ByteBuffer.allocateDirect(resampledSize).order(ByteOrder.nativeOrder()); + } else { + buffer.clear(); + } + + // Samples are little endian. + switch (sourceEncoding) { + case C.ENCODING_PCM_24BIT: + // 24->32 bit resampling. + for (int i = offset; i < limit; i += 3) { + int val = (inputBuffer.get(i) << 8) & 0x0000ff00 | (inputBuffer.get(i + 1) << 16) & 0x00ff0000 | + (inputBuffer.get(i + 2) << 24) & 0xff000000; + writePcm32bitFloat(val, buffer); + } + break; + case C.ENCODING_PCM_32BIT: + case C.ENCODING_PCM_8BIT: + case C.ENCODING_PCM_16BIT: + case C.ENCODING_PCM_FLOAT: + case C.ENCODING_INVALID: + case Format.NO_VALUE: + default: + // Never happens. + throw new IllegalStateException(); + } + + inputBuffer.position(inputBuffer.limit()); + buffer.flip(); + outputBuffer = buffer; + } + + @Override + public void queueEndOfStream() { + inputEnded = true; + } + + @Override + public ByteBuffer getOutput() { + ByteBuffer outputBuffer = this.outputBuffer; + this.outputBuffer = EMPTY_BUFFER; + return outputBuffer; + } + + @SuppressWarnings("ReferenceEquality") + @Override + public boolean isEnded() { + return inputEnded && outputBuffer == EMPTY_BUFFER; + } + + @Override + public void flush() { + outputBuffer = EMPTY_BUFFER; + inputEnded = false; + } + + @Override + public void reset() { + flush(); + buffer = EMPTY_BUFFER; + sampleRateHz = Format.NO_VALUE; + channelCount = Format.NO_VALUE; + sourceEncoding = C.ENCODING_INVALID; + } + + /** + * Converts the provided value into 32-bit float PCM and writes to buffer. + * + * @param val 32-bit int value to convert to 32-bit float [-1.0, 1.0] + * @param buffer The output buffer. + */ + private static void writePcm32bitFloat(int val, ByteBuffer buffer) { + float convVal = (float) (PCM_INT32_FLOAT * val); + int bits = Float.floatToIntBits(convVal); + if (bits == 0x7fc00000) + bits = Float.floatToIntBits((float) 0.0); + buffer.put((byte) (bits & 0xff)); + buffer.put((byte) ((bits >> 8) & 0xff)); + buffer.put((byte) ((bits >> 16) & 0xff)); + buffer.put((byte) ((bits >> 24) & 0xff)); + } + +} \ No newline at end of file diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index f73d63616b4..d5e1b6ab03b 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -58,6 +58,9 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media private int encoderPadding; private long currentPositionUs; private boolean allowPositionDiscontinuity; + private final boolean dontDither24bitPCM; + private ByteBuffer resampledBuffer; + private FloatResamplingAudioProcessor floatResamplingAudioProcessor; /** * @param mediaCodecSelector A decoder selector. @@ -137,7 +140,37 @@ public MediaCodecAudioRenderer(MediaCodecSelector mediaCodecSelector, @Nullable AudioRendererEventListener eventListener, @Nullable AudioCapabilities audioCapabilities, AudioProcessor... audioProcessors) { this(mediaCodecSelector, drmSessionManager, playClearSamplesWithoutKeys, - eventHandler, eventListener, new DefaultAudioSink(audioCapabilities, audioProcessors)); + eventHandler, eventListener, new DefaultAudioSink(audioCapabilities, audioProcessors), + false); + } + + /** + * @param mediaCodecSelector A decoder selector. + * @param drmSessionManager For use with encrypted content. May be null if support for encrypted + * content is not required. + * @param playClearSamplesWithoutKeys Encrypted media may contain clear (un-encrypted) regions. + * For example a media file may start with a short clear region so as to allow playback to + * begin in parallel with key acquisition. This parameter specifies whether the renderer is + * permitted to play clear regions of encrypted media files before {@code drmSessionManager} + * has obtained the keys necessary to decrypt encrypted regions of the media. + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param audioCapabilities The audio capabilities for playback on this device. May be null if the + * default capabilities (no encoded audio passthrough support) should be assumed. + * @param dontDither24bitPCM If the input is 24bit PCM audio convert to 32bit Float PCM + * @param audioProcessors Optional {@link AudioProcessor}s that will process PCM audio before + * output. + */ + public MediaCodecAudioRenderer(MediaCodecSelector mediaCodecSelector, + @Nullable DrmSessionManager drmSessionManager, + boolean playClearSamplesWithoutKeys, @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + @Nullable AudioCapabilities audioCapabilities, boolean dontDither24bitPCM, + AudioProcessor... audioProcessors) { + this(mediaCodecSelector, drmSessionManager, playClearSamplesWithoutKeys, + eventHandler, eventListener, new DefaultAudioSink(audioCapabilities, audioProcessors), + dontDither24bitPCM); } /** @@ -158,9 +191,34 @@ public MediaCodecAudioRenderer(MediaCodecSelector mediaCodecSelector, @Nullable DrmSessionManager drmSessionManager, boolean playClearSamplesWithoutKeys, @Nullable Handler eventHandler, @Nullable AudioRendererEventListener eventListener, AudioSink audioSink) { + this(mediaCodecSelector, drmSessionManager, playClearSamplesWithoutKeys, eventHandler, + eventListener, audioSink, false); + } + + /** + * @param mediaCodecSelector A decoder selector. + * @param drmSessionManager For use with encrypted content. May be null if support for encrypted + * content is not required. + * @param playClearSamplesWithoutKeys Encrypted media may contain clear (un-encrypted) regions. + * For example a media file may start with a short clear region so as to allow playback to + * begin in parallel with key acquisition. This parameter specifies whether the renderer is + * permitted to play clear regions of encrypted media files before {@code drmSessionManager} + * has obtained the keys necessary to decrypt encrypted regions of the media. + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param audioSink The sink to which audio will be output. + * @param dontDither24bitPCM If the input is 24bit PCM audio convert to 32bit Float PCM + */ + public MediaCodecAudioRenderer(MediaCodecSelector mediaCodecSelector, + @Nullable DrmSessionManager drmSessionManager, + boolean playClearSamplesWithoutKeys, @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, AudioSink audioSink, + boolean dontDither24bitPCM) { super(C.TRACK_TYPE_AUDIO, mediaCodecSelector, drmSessionManager, playClearSamplesWithoutKeys); eventDispatcher = new EventDispatcher(eventHandler, eventListener); this.audioSink = audioSink; + this.dontDither24bitPCM = dontDither24bitPCM; audioSink.setListener(new AudioSinkListener()); } @@ -268,10 +326,20 @@ protected void onCodecInitialized(String name, long initializedTimestampMs, protected void onInputFormatChanged(Format newFormat) throws ExoPlaybackException { super.onInputFormatChanged(newFormat); eventDispatcher.inputFormatChanged(newFormat); - // If the input format is anything other than PCM then we assume that the audio decoder will - // output 16-bit PCM. - pcmEncoding = MimeTypes.AUDIO_RAW.equals(newFormat.sampleMimeType) ? newFormat.pcmEncoding - : C.ENCODING_PCM_16BIT; + + // if the input is 24bit pcm audio and we explicitly said not to dither then convert it to float + if (dontDither24bitPCM && newFormat.pcmEncoding == C.ENCODING_PCM_24BIT) { + if (floatResamplingAudioProcessor == null) + floatResamplingAudioProcessor = new FloatResamplingAudioProcessor(); + pcmEncoding = floatResamplingAudioProcessor.getOutputEncoding(); + } else { + // If the input format is anything other than PCM then we assume that the audio decoder will + // output 16-bit PCM. + pcmEncoding = MimeTypes.AUDIO_RAW.equals(newFormat.sampleMimeType) ? newFormat.pcmEncoding + : C.ENCODING_PCM_16BIT; + floatResamplingAudioProcessor = null; + } + channelCount = newFormat.channelCount; encoderDelay = newFormat.encoderDelay != Format.NO_VALUE ? newFormat.encoderDelay : 0; encoderPadding = newFormat.encoderPadding != Format.NO_VALUE ? newFormat.encoderPadding : 0; @@ -302,9 +370,11 @@ protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputFormat) } try { + if (floatResamplingAudioProcessor != null) + floatResamplingAudioProcessor.configure(sampleRate, channelCount, C.ENCODING_PCM_24BIT); audioSink.configure(encoding, channelCount, sampleRate, 0, channelMap, encoderDelay, encoderPadding); - } catch (AudioSink.ConfigurationException e) { + } catch (AudioSink.ConfigurationException | AudioProcessor.UnhandledFormatException e) { throw ExoPlaybackException.createForRenderer(e, getIndex()); } } @@ -420,19 +490,35 @@ protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, M codec.releaseOutputBuffer(bufferIndex, false); return true; } - if (shouldSkip) { codec.releaseOutputBuffer(bufferIndex, false); decoderCounters.skippedOutputBufferCount++; audioSink.handleDiscontinuity(); + resampledBuffer = null; return true; } try { - if (audioSink.handleBuffer(buffer, bufferPresentationTimeUs)) { - codec.releaseOutputBuffer(bufferIndex, false); - decoderCounters.renderedOutputBufferCount++; - return true; + if (floatResamplingAudioProcessor != null) { + boolean draining = resampledBuffer != null; + if (!draining) { + floatResamplingAudioProcessor.queueInput(buffer); + resampledBuffer = floatResamplingAudioProcessor.getOutput(); + } + if (audioSink.handleBuffer(resampledBuffer, bufferPresentationTimeUs)) + resampledBuffer = null; + if (!draining) { + codec.releaseOutputBuffer(bufferIndex, false); + decoderCounters.renderedOutputBufferCount++; + return true; + } + } + else { + if (audioSink.handleBuffer(buffer, bufferPresentationTimeUs)) { + codec.releaseOutputBuffer(bufferIndex, false); + decoderCounters.renderedOutputBufferCount++; + return true; + } } } catch (AudioSink.InitializationException | AudioSink.WriteException e) { throw ExoPlaybackException.createForRenderer(e, getIndex()); From 821ea0e58b94ffa2eb95a7934c173aa44752fe2a Mon Sep 17 00:00:00 2001 From: Drew Hill Date: Sat, 6 Jan 2018 00:26:18 -0500 Subject: [PATCH 2/3] moved floatresample into defaultaudiosink and added new constructor in defaultaudiosync to use that resample when audio input is 24/32bit pcm and the new flag is enabled --- .../exoplayer2/audio/DefaultAudioSink.java | 39 ++++++- .../audio/FloatResamplingAudioProcessor.java | 15 ++- .../audio/MediaCodecAudioRenderer.java | 108 ++---------------- 3 files changed, 60 insertions(+), 102 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java index e3bf72c541a..ee5ca17bc7a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java @@ -164,10 +164,12 @@ public InvalidAudioTrackTimestampException(String message) { public static boolean failOnSpuriousAudioTimestamp = false; @Nullable private final AudioCapabilities audioCapabilities; + private final boolean canConvertHiResPcmToFloat; private final ChannelMappingAudioProcessor channelMappingAudioProcessor; private final TrimmingAudioProcessor trimmingAudioProcessor; private final SonicAudioProcessor sonicAudioProcessor; private final AudioProcessor[] availableAudioProcessors; + private final AudioProcessor[] hiResAvailableAudioProcessors; private final ConditionVariable releasingConditionVariable; private final long[] playheadOffsets; private final AudioTrackUtil audioTrackUtil; @@ -180,6 +182,7 @@ public InvalidAudioTrackTimestampException(String message) { private AudioTrack keepSessionIdAudioTrack; private AudioTrack audioTrack; private boolean isInputPcm; + private boolean shouldUpResPCMAudio; private int inputSampleRate; private int sampleRate; private int channelConfig; @@ -233,6 +236,8 @@ public InvalidAudioTrackTimestampException(String message) { private boolean hasData; private long lastFeedElapsedRealtimeMs; + + /** * @param audioCapabilities The audio capabilities for playback on this device. May be null if the * default capabilities (no encoded audio passthrough support) should be assumed. @@ -241,7 +246,23 @@ public InvalidAudioTrackTimestampException(String message) { */ public DefaultAudioSink(@Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors) { + this(audioCapabilities, audioProcessors, false); + } + + /** + * @param audioCapabilities The audio capabilities for playback on this device. May be null if the + * default capabilities (no encoded audio passthrough support) should be assumed. + * @param audioProcessors An array of {@link AudioProcessor}s that will process PCM audio before + * output. May be empty. + * @param canConvertHiResPcmToFloat Flag to convert > 16bit PCM Audio to 32bit Float PCM Audio to + * avoid dithering the input audio. If enabled other audio processors that expect 16bit PCM + * are disabled + */ + public DefaultAudioSink(@Nullable AudioCapabilities audioCapabilities, + AudioProcessor[] audioProcessors, boolean canConvertHiResPcmToFloat) { + this.audioCapabilities = audioCapabilities; + this.canConvertHiResPcmToFloat = canConvertHiResPcmToFloat; releasingConditionVariable = new ConditionVariable(true); if (Util.SDK_INT >= 18) { try { @@ -265,6 +286,8 @@ public DefaultAudioSink(@Nullable AudioCapabilities audioCapabilities, availableAudioProcessors[2] = trimmingAudioProcessor; System.arraycopy(audioProcessors, 0, availableAudioProcessors, 3, audioProcessors.length); availableAudioProcessors[3 + audioProcessors.length] = sonicAudioProcessor; + hiResAvailableAudioProcessors = new AudioProcessor[1]; + hiResAvailableAudioProcessors[0] = new FloatResamplingAudioProcessor(); playheadOffsets = new long[MAX_PLAYHEAD_OFFSET_COUNT]; volume = 1.0f; startMediaTimeState = START_NOT_SET; @@ -342,15 +365,20 @@ public void configure(@C.Encoding int inputEncoding, int inputChannelCount, int int channelCount = inputChannelCount; int sampleRate = inputSampleRate; isInputPcm = isEncodingPcm(inputEncoding); + shouldUpResPCMAudio = canConvertHiResPcmToFloat && + (inputEncoding == C.ENCODING_PCM_24BIT || inputEncoding == C.ENCODING_PCM_32BIT); if (isInputPcm) { - pcmFrameSize = Util.getPcmFrameSize(inputEncoding, channelCount); + pcmFrameSize = Util.getPcmFrameSize(shouldUpResPCMAudio + ? C.ENCODING_PCM_FLOAT : inputEncoding, channelCount); } @C.Encoding int encoding = inputEncoding; boolean processingEnabled = isInputPcm && inputEncoding != C.ENCODING_PCM_FLOAT; if (processingEnabled) { + AudioProcessor[] activeAudioProcessors = shouldUpResPCMAudio ? + hiResAvailableAudioProcessors : availableAudioProcessors; trimmingAudioProcessor.setTrimSampleCount(trimStartSamples, trimEndSamples); channelMappingAudioProcessor.setChannelMap(outputChannels); - for (AudioProcessor audioProcessor : availableAudioProcessors) { + for (AudioProcessor audioProcessor : activeAudioProcessors) { try { flush |= audioProcessor.configure(sampleRate, channelCount, encoding); } catch (AudioProcessor.UnhandledFormatException e) { @@ -460,7 +488,9 @@ public void configure(@C.Encoding int inputEncoding, int inputChannelCount, int private void resetAudioProcessors() { ArrayList newAudioProcessors = new ArrayList<>(); - for (AudioProcessor audioProcessor : availableAudioProcessors) { + AudioProcessor[] activeAudioProcessors = shouldUpResPCMAudio ? + hiResAvailableAudioProcessors : availableAudioProcessors; + for (AudioProcessor audioProcessor : activeAudioProcessors) { if (audioProcessor.isActive()) { newAudioProcessors.add(audioProcessor); } else { @@ -967,6 +997,9 @@ public void release() { for (AudioProcessor audioProcessor : availableAudioProcessors) { audioProcessor.reset(); } + for (AudioProcessor audioProcessor : hiResAvailableAudioProcessors) { + audioProcessor.reset(); + } audioSessionId = C.AUDIO_SESSION_ID_UNSET; playing = false; } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java index 28d2eca25f9..b0f48d43fa0 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java @@ -36,7 +36,7 @@ public FloatResamplingAudioProcessor() { @Override public boolean configure(int sampleRateHz, int channelCount, @C.Encoding int encoding) throws AudioProcessor.UnhandledFormatException { - if (encoding != C.ENCODING_PCM_24BIT) { + if (encoding != C.ENCODING_PCM_24BIT && encoding != C.ENCODING_PCM_32BIT) { throw new AudioProcessor.UnhandledFormatException(sampleRateHz, channelCount, encoding); } if (this.sampleRateHz == sampleRateHz && this.channelCount == channelCount @@ -51,7 +51,9 @@ public boolean configure(int sampleRateHz, int channelCount, @C.Encoding int enc } @Override - public boolean isActive() { return sourceEncoding == C.ENCODING_PCM_24BIT; } + public boolean isActive() { + return sourceEncoding == C.ENCODING_PCM_24BIT || sourceEncoding == C.ENCODING_PCM_32BIT; + } @Override public int getOutputChannelCount() { return channelCount; } @@ -76,6 +78,8 @@ public void queueInput(ByteBuffer inputBuffer) { resampledSize = (size / 3) * 4; break; case C.ENCODING_PCM_32BIT: + resampledSize = size; + break; case C.ENCODING_PCM_8BIT: case C.ENCODING_PCM_16BIT: case C.ENCODING_PCM_FLOAT: @@ -103,6 +107,13 @@ public void queueInput(ByteBuffer inputBuffer) { } break; case C.ENCODING_PCM_32BIT: + // 32->32 bit conversion. + for (int i = offset; i < limit; i += 4) { + int val = inputBuffer.get(i) & 0x000000ff | (inputBuffer.get(i) << 8) & 0x0000ff00 | + (inputBuffer.get(i + 1) << 16) & 0x00ff0000 | (inputBuffer.get(i + 2) << 24) & 0xff000000; + writePcm32bitFloat(val, buffer); + } + break; case C.ENCODING_PCM_8BIT: case C.ENCODING_PCM_16BIT: case C.ENCODING_PCM_FLOAT: diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index d5e1b6ab03b..f73d63616b4 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -58,9 +58,6 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media private int encoderPadding; private long currentPositionUs; private boolean allowPositionDiscontinuity; - private final boolean dontDither24bitPCM; - private ByteBuffer resampledBuffer; - private FloatResamplingAudioProcessor floatResamplingAudioProcessor; /** * @param mediaCodecSelector A decoder selector. @@ -140,37 +137,7 @@ public MediaCodecAudioRenderer(MediaCodecSelector mediaCodecSelector, @Nullable AudioRendererEventListener eventListener, @Nullable AudioCapabilities audioCapabilities, AudioProcessor... audioProcessors) { this(mediaCodecSelector, drmSessionManager, playClearSamplesWithoutKeys, - eventHandler, eventListener, new DefaultAudioSink(audioCapabilities, audioProcessors), - false); - } - - /** - * @param mediaCodecSelector A decoder selector. - * @param drmSessionManager For use with encrypted content. May be null if support for encrypted - * content is not required. - * @param playClearSamplesWithoutKeys Encrypted media may contain clear (un-encrypted) regions. - * For example a media file may start with a short clear region so as to allow playback to - * begin in parallel with key acquisition. This parameter specifies whether the renderer is - * permitted to play clear regions of encrypted media files before {@code drmSessionManager} - * has obtained the keys necessary to decrypt encrypted regions of the media. - * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be - * null if delivery of events is not required. - * @param eventListener A listener of events. May be null if delivery of events is not required. - * @param audioCapabilities The audio capabilities for playback on this device. May be null if the - * default capabilities (no encoded audio passthrough support) should be assumed. - * @param dontDither24bitPCM If the input is 24bit PCM audio convert to 32bit Float PCM - * @param audioProcessors Optional {@link AudioProcessor}s that will process PCM audio before - * output. - */ - public MediaCodecAudioRenderer(MediaCodecSelector mediaCodecSelector, - @Nullable DrmSessionManager drmSessionManager, - boolean playClearSamplesWithoutKeys, @Nullable Handler eventHandler, - @Nullable AudioRendererEventListener eventListener, - @Nullable AudioCapabilities audioCapabilities, boolean dontDither24bitPCM, - AudioProcessor... audioProcessors) { - this(mediaCodecSelector, drmSessionManager, playClearSamplesWithoutKeys, - eventHandler, eventListener, new DefaultAudioSink(audioCapabilities, audioProcessors), - dontDither24bitPCM); + eventHandler, eventListener, new DefaultAudioSink(audioCapabilities, audioProcessors)); } /** @@ -191,34 +158,9 @@ public MediaCodecAudioRenderer(MediaCodecSelector mediaCodecSelector, @Nullable DrmSessionManager drmSessionManager, boolean playClearSamplesWithoutKeys, @Nullable Handler eventHandler, @Nullable AudioRendererEventListener eventListener, AudioSink audioSink) { - this(mediaCodecSelector, drmSessionManager, playClearSamplesWithoutKeys, eventHandler, - eventListener, audioSink, false); - } - - /** - * @param mediaCodecSelector A decoder selector. - * @param drmSessionManager For use with encrypted content. May be null if support for encrypted - * content is not required. - * @param playClearSamplesWithoutKeys Encrypted media may contain clear (un-encrypted) regions. - * For example a media file may start with a short clear region so as to allow playback to - * begin in parallel with key acquisition. This parameter specifies whether the renderer is - * permitted to play clear regions of encrypted media files before {@code drmSessionManager} - * has obtained the keys necessary to decrypt encrypted regions of the media. - * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be - * null if delivery of events is not required. - * @param eventListener A listener of events. May be null if delivery of events is not required. - * @param audioSink The sink to which audio will be output. - * @param dontDither24bitPCM If the input is 24bit PCM audio convert to 32bit Float PCM - */ - public MediaCodecAudioRenderer(MediaCodecSelector mediaCodecSelector, - @Nullable DrmSessionManager drmSessionManager, - boolean playClearSamplesWithoutKeys, @Nullable Handler eventHandler, - @Nullable AudioRendererEventListener eventListener, AudioSink audioSink, - boolean dontDither24bitPCM) { super(C.TRACK_TYPE_AUDIO, mediaCodecSelector, drmSessionManager, playClearSamplesWithoutKeys); eventDispatcher = new EventDispatcher(eventHandler, eventListener); this.audioSink = audioSink; - this.dontDither24bitPCM = dontDither24bitPCM; audioSink.setListener(new AudioSinkListener()); } @@ -326,20 +268,10 @@ protected void onCodecInitialized(String name, long initializedTimestampMs, protected void onInputFormatChanged(Format newFormat) throws ExoPlaybackException { super.onInputFormatChanged(newFormat); eventDispatcher.inputFormatChanged(newFormat); - - // if the input is 24bit pcm audio and we explicitly said not to dither then convert it to float - if (dontDither24bitPCM && newFormat.pcmEncoding == C.ENCODING_PCM_24BIT) { - if (floatResamplingAudioProcessor == null) - floatResamplingAudioProcessor = new FloatResamplingAudioProcessor(); - pcmEncoding = floatResamplingAudioProcessor.getOutputEncoding(); - } else { - // If the input format is anything other than PCM then we assume that the audio decoder will - // output 16-bit PCM. - pcmEncoding = MimeTypes.AUDIO_RAW.equals(newFormat.sampleMimeType) ? newFormat.pcmEncoding - : C.ENCODING_PCM_16BIT; - floatResamplingAudioProcessor = null; - } - + // If the input format is anything other than PCM then we assume that the audio decoder will + // output 16-bit PCM. + pcmEncoding = MimeTypes.AUDIO_RAW.equals(newFormat.sampleMimeType) ? newFormat.pcmEncoding + : C.ENCODING_PCM_16BIT; channelCount = newFormat.channelCount; encoderDelay = newFormat.encoderDelay != Format.NO_VALUE ? newFormat.encoderDelay : 0; encoderPadding = newFormat.encoderPadding != Format.NO_VALUE ? newFormat.encoderPadding : 0; @@ -370,11 +302,9 @@ protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputFormat) } try { - if (floatResamplingAudioProcessor != null) - floatResamplingAudioProcessor.configure(sampleRate, channelCount, C.ENCODING_PCM_24BIT); audioSink.configure(encoding, channelCount, sampleRate, 0, channelMap, encoderDelay, encoderPadding); - } catch (AudioSink.ConfigurationException | AudioProcessor.UnhandledFormatException e) { + } catch (AudioSink.ConfigurationException e) { throw ExoPlaybackException.createForRenderer(e, getIndex()); } } @@ -490,35 +420,19 @@ protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, M codec.releaseOutputBuffer(bufferIndex, false); return true; } + if (shouldSkip) { codec.releaseOutputBuffer(bufferIndex, false); decoderCounters.skippedOutputBufferCount++; audioSink.handleDiscontinuity(); - resampledBuffer = null; return true; } try { - if (floatResamplingAudioProcessor != null) { - boolean draining = resampledBuffer != null; - if (!draining) { - floatResamplingAudioProcessor.queueInput(buffer); - resampledBuffer = floatResamplingAudioProcessor.getOutput(); - } - if (audioSink.handleBuffer(resampledBuffer, bufferPresentationTimeUs)) - resampledBuffer = null; - if (!draining) { - codec.releaseOutputBuffer(bufferIndex, false); - decoderCounters.renderedOutputBufferCount++; - return true; - } - } - else { - if (audioSink.handleBuffer(buffer, bufferPresentationTimeUs)) { - codec.releaseOutputBuffer(bufferIndex, false); - decoderCounters.renderedOutputBufferCount++; - return true; - } + if (audioSink.handleBuffer(buffer, bufferPresentationTimeUs)) { + codec.releaseOutputBuffer(bufferIndex, false); + decoderCounters.renderedOutputBufferCount++; + return true; } } catch (AudioSink.InitializationException | AudioSink.WriteException e) { throw ExoPlaybackException.createForRenderer(e, getIndex()); From aaf469ce065205593e99cfbd655e6f57b7763d62 Mon Sep 17 00:00:00 2001 From: Drew Hill Date: Sun, 14 Jan 2018 10:55:54 -0500 Subject: [PATCH 3/3] code review changes and fix for discontinuity --- .../exoplayer2/audio/DefaultAudioSink.java | 43 ++++++++++--------- .../audio/FloatResamplingAudioProcessor.java | 20 +++++++-- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java index ee5ca17bc7a..892fd428ff8 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java @@ -168,8 +168,8 @@ public InvalidAudioTrackTimestampException(String message) { private final ChannelMappingAudioProcessor channelMappingAudioProcessor; private final TrimmingAudioProcessor trimmingAudioProcessor; private final SonicAudioProcessor sonicAudioProcessor; - private final AudioProcessor[] availableAudioProcessors; - private final AudioProcessor[] hiResAvailableAudioProcessors; + private final AudioProcessor[] toIntPcmAvailableAudioProcessors; + private final AudioProcessor[] toFloatPcmAvailableAudioProcessors; private final ConditionVariable releasingConditionVariable; private final long[] playheadOffsets; private final AudioTrackUtil audioTrackUtil; @@ -189,6 +189,7 @@ public InvalidAudioTrackTimestampException(String message) { private @C.Encoding int outputEncoding; private AudioAttributes audioAttributes; private boolean processingEnabled; + private boolean canApplyPlaybackParams; private int bufferSize; private long bufferSizeUs; @@ -280,14 +281,14 @@ public DefaultAudioSink(@Nullable AudioCapabilities audioCapabilities, channelMappingAudioProcessor = new ChannelMappingAudioProcessor(); trimmingAudioProcessor = new TrimmingAudioProcessor(); sonicAudioProcessor = new SonicAudioProcessor(); - availableAudioProcessors = new AudioProcessor[4 + audioProcessors.length]; - availableAudioProcessors[0] = new ResamplingAudioProcessor(); - availableAudioProcessors[1] = channelMappingAudioProcessor; - availableAudioProcessors[2] = trimmingAudioProcessor; - System.arraycopy(audioProcessors, 0, availableAudioProcessors, 3, audioProcessors.length); - availableAudioProcessors[3 + audioProcessors.length] = sonicAudioProcessor; - hiResAvailableAudioProcessors = new AudioProcessor[1]; - hiResAvailableAudioProcessors[0] = new FloatResamplingAudioProcessor(); + toIntPcmAvailableAudioProcessors = new AudioProcessor[4 + audioProcessors.length]; + toIntPcmAvailableAudioProcessors[0] = new ResamplingAudioProcessor(); + toIntPcmAvailableAudioProcessors[1] = channelMappingAudioProcessor; + toIntPcmAvailableAudioProcessors[2] = trimmingAudioProcessor; + System.arraycopy(audioProcessors, 0, toIntPcmAvailableAudioProcessors, 3, audioProcessors.length); + toIntPcmAvailableAudioProcessors[3 + audioProcessors.length] = sonicAudioProcessor; + toFloatPcmAvailableAudioProcessors = new AudioProcessor[1]; + toFloatPcmAvailableAudioProcessors[0] = new FloatResamplingAudioProcessor(); playheadOffsets = new long[MAX_PLAYHEAD_OFFSET_COUNT]; volume = 1.0f; startMediaTimeState = START_NOT_SET; @@ -368,17 +369,17 @@ public void configure(@C.Encoding int inputEncoding, int inputChannelCount, int shouldUpResPCMAudio = canConvertHiResPcmToFloat && (inputEncoding == C.ENCODING_PCM_24BIT || inputEncoding == C.ENCODING_PCM_32BIT); if (isInputPcm) { - pcmFrameSize = Util.getPcmFrameSize(shouldUpResPCMAudio - ? C.ENCODING_PCM_FLOAT : inputEncoding, channelCount); + pcmFrameSize = Util.getPcmFrameSize(inputEncoding, channelCount); } @C.Encoding int encoding = inputEncoding; boolean processingEnabled = isInputPcm && inputEncoding != C.ENCODING_PCM_FLOAT; + canApplyPlaybackParams = processingEnabled && !shouldUpResPCMAudio; if (processingEnabled) { - AudioProcessor[] activeAudioProcessors = shouldUpResPCMAudio ? - hiResAvailableAudioProcessors : availableAudioProcessors; + AudioProcessor[] availableAudioProcessors = shouldUpResPCMAudio ? + toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors; trimmingAudioProcessor.setTrimSampleCount(trimStartSamples, trimEndSamples); channelMappingAudioProcessor.setChannelMap(outputChannels); - for (AudioProcessor audioProcessor : activeAudioProcessors) { + for (AudioProcessor audioProcessor : availableAudioProcessors) { try { flush |= audioProcessor.configure(sampleRate, channelCount, encoding); } catch (AudioProcessor.UnhandledFormatException e) { @@ -488,9 +489,9 @@ public void configure(@C.Encoding int inputEncoding, int inputChannelCount, int private void resetAudioProcessors() { ArrayList newAudioProcessors = new ArrayList<>(); - AudioProcessor[] activeAudioProcessors = shouldUpResPCMAudio ? - hiResAvailableAudioProcessors : availableAudioProcessors; - for (AudioProcessor audioProcessor : activeAudioProcessors) { + AudioProcessor[] availableAudioProcessors = shouldUpResPCMAudio ? + toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors; + for (AudioProcessor audioProcessor : availableAudioProcessors) { if (audioProcessor.isActive()) { newAudioProcessors.add(audioProcessor); } else { @@ -838,7 +839,7 @@ public boolean hasPendingData() { @Override public PlaybackParameters setPlaybackParameters(PlaybackParameters playbackParameters) { - if (isInitialized() && !processingEnabled) { + if (isInitialized() && !canApplyPlaybackParams) { // The playback parameters are always the default if processing is disabled. this.playbackParameters = PlaybackParameters.DEFAULT; return this.playbackParameters; @@ -994,10 +995,10 @@ public void run() { public void release() { reset(); releaseKeepSessionIdAudioTrack(); - for (AudioProcessor audioProcessor : availableAudioProcessors) { + for (AudioProcessor audioProcessor : toIntPcmAvailableAudioProcessors) { audioProcessor.reset(); } - for (AudioProcessor audioProcessor : hiResAvailableAudioProcessors) { + for (AudioProcessor audioProcessor : toFloatPcmAvailableAudioProcessors) { audioProcessor.reset(); } audioSessionId = C.AUDIO_SESSION_ID_UNSET; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java index b0f48d43fa0..f7073f12759 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.android.exoplayer2.audio; @@ -173,10 +188,7 @@ private static void writePcm32bitFloat(int val, ByteBuffer buffer) { int bits = Float.floatToIntBits(convVal); if (bits == 0x7fc00000) bits = Float.floatToIntBits((float) 0.0); - buffer.put((byte) (bits & 0xff)); - buffer.put((byte) ((bits >> 8) & 0xff)); - buffer.put((byte) ((bits >> 16) & 0xff)); - buffer.put((byte) ((bits >> 24) & 0xff)); + buffer.putInt(bits); } } \ No newline at end of file