From 374a89a9fbbeb98e72fb3feaf51d022c71a15334 Mon Sep 17 00:00:00 2001 From: dalgwen Date: Sat, 11 Dec 2021 13:08:03 +0100 Subject: [PATCH] [pulseaudio] Add reencoding to play more audio formats (#11630) (#11631) Add a pass to reencode PCM sound in 16 bit, 44100 hz, 2 channels, before sending it to the pulseaudio audio sink. Signed-off-by: Gwendal Roulleau Co-authored-by: Gwendal Roulleau --- .../internal/ConvertedInputStream.java | 257 ++++++++++++++++++ .../internal/PulseAudioAudioSink.java | 118 +------- 2 files changed, 267 insertions(+), 108 deletions(-) create mode 100644 bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/ConvertedInputStream.java diff --git a/bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/ConvertedInputStream.java b/bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/ConvertedInputStream.java new file mode 100644 index 0000000000000..66b902bc9cfcb --- /dev/null +++ b/bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/ConvertedInputStream.java @@ -0,0 +1,257 @@ +/** + * Copyright (c) 2010-2021 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.binding.pulseaudio.internal; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import javazoom.spi.mpeg.sampled.convert.MpegFormatConversionProvider; +import javazoom.spi.mpeg.sampled.file.MpegAudioFileReader; + +import javax.sound.sampled.AudioFileFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.audio.AudioFormat; +import org.openhab.core.audio.AudioStream; +import org.openhab.core.audio.FixedLengthAudioStream; +import org.openhab.core.audio.UnsupportedAudioFormatException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tritonus.share.sampled.file.TAudioFileFormat; + +/** + * This class convert a stream to the normalized pcm + * format wanted by the pulseaudio sink + * + * @author Gwendal Roulleau - Initial contribution + */ +@NonNullByDefault +public class ConvertedInputStream extends InputStream { + + private final Logger logger = LoggerFactory.getLogger(ConvertedInputStream.class); + + private static final javax.sound.sampled.AudioFormat TARGET_FORMAT = new javax.sound.sampled.AudioFormat( + javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED, 44100, 16, 2, 4, 44100, false); + + private final AudioFormat audioFormat; + private AudioInputStream pcmNormalizedInputStream; + + private long duration = -1; + private long length = -1; + + public ConvertedInputStream(AudioStream innerInputStream) + throws UnsupportedAudioFormatException, UnsupportedAudioFileException, IOException { + + this.audioFormat = innerInputStream.getFormat(); + + if (innerInputStream instanceof FixedLengthAudioStream) { + length = ((FixedLengthAudioStream) innerInputStream).length(); + } + + pcmNormalizedInputStream = getPCMStreamNormalized(getPCMStream(new ResetableInputStream(innerInputStream))); + } + + @Override + public int read(byte @Nullable [] b) throws IOException { + return pcmNormalizedInputStream.read(b); + } + + @Override + public int read(byte @Nullable [] b, int off, int len) throws IOException { + return pcmNormalizedInputStream.read(b, off, len); + } + + @Override + public byte[] readAllBytes() throws IOException { + return pcmNormalizedInputStream.readAllBytes(); + } + + @Override + public byte[] readNBytes(int len) throws IOException { + return pcmNormalizedInputStream.readNBytes(len); + } + + @Override + public int readNBytes(byte @Nullable [] b, int off, int len) throws IOException { + return pcmNormalizedInputStream.readNBytes(b, off, len); + } + + @Override + public int read() throws IOException { + return pcmNormalizedInputStream.read(); + } + + @Override + public void close() throws IOException { + pcmNormalizedInputStream.close(); + } + + /** + * Ensure right PCM format by converting if needed (sample rate, channel) + * + * @param pcmInputStream + * + * @return A PCM normalized stream (2 channel, 44100hz, 16 bit signed) + */ + private AudioInputStream getPCMStreamNormalized(AudioInputStream pcmInputStream) { + + javax.sound.sampled.AudioFormat format = pcmInputStream.getFormat(); + if (format.getChannels() != 2 + || !format.getEncoding().equals(javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED) + || Math.abs(format.getFrameRate() - 44100) > 1000) { + logger.debug("Sound is not in the target format. Trying to reencode it"); + return AudioSystem.getAudioInputStream(TARGET_FORMAT, pcmInputStream); + } else { + return pcmInputStream; + } + } + + public long getDuration() { + return duration; + } + + /** + * If necessary, this method convert MP3 to PCM, and try to + * extract duration information. + * + * @param resetableInnerInputStream A stream supporting reset operation + * (reset is mandatory to parse formation without loosing data) + * + * @return PCM stream + * @throws UnsupportedAudioFileException + * @throws IOException + * @throws UnsupportedAudioFormatException + */ + private AudioInputStream getPCMStream(InputStream resetableInnerInputStream) + throws UnsupportedAudioFileException, IOException, UnsupportedAudioFormatException { + + if (AudioFormat.MP3.isCompatible(audioFormat)) { + MpegAudioFileReader mpegAudioFileReader = new MpegAudioFileReader(); + + if (length > 0) { // compute duration if possible + AudioFileFormat audioFileFormat = mpegAudioFileReader.getAudioFileFormat(resetableInnerInputStream); + if (audioFileFormat instanceof TAudioFileFormat) { + Map taudioFileFormatProperties = ((TAudioFileFormat) audioFileFormat).properties(); + if (taudioFileFormatProperties.containsKey("mp3.framesize.bytes") + && taudioFileFormatProperties.containsKey("mp3.framerate.fps")) { + Integer frameSize = (Integer) taudioFileFormatProperties.get("mp3.framesize.bytes"); + Float frameRate = (Float) taudioFileFormatProperties.get("mp3.framerate.fps"); + if (frameSize != null && frameRate != null) { + duration = Math.round((length / (frameSize * frameRate)) * 1000); + logger.debug("Duration of input stream : {}", duration); + } + } + } + resetableInnerInputStream.reset(); + } + + logger.debug("Sound is a MP3. Trying to reencode it"); + // convert MP3 to PCM : + AudioInputStream sourceAIS = mpegAudioFileReader.getAudioInputStream(resetableInnerInputStream); + javax.sound.sampled.AudioFormat sourceFormat = sourceAIS.getFormat(); + + MpegFormatConversionProvider mpegconverter = new MpegFormatConversionProvider(); + javax.sound.sampled.AudioFormat convertFormat = new javax.sound.sampled.AudioFormat( + javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED, sourceFormat.getSampleRate(), 16, + sourceFormat.getChannels(), sourceFormat.getChannels() * 2, sourceFormat.getSampleRate(), false); + + return mpegconverter.getAudioInputStream(convertFormat, sourceAIS); + + } else if (AudioFormat.WAV.isCompatible(audioFormat)) { + // return the same input stream, but try to compute the duration first + AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(resetableInnerInputStream); + if (length > 0) { + int frameSize = audioInputStream.getFormat().getFrameSize(); + float frameRate = audioInputStream.getFormat().getFrameRate(); + float durationInSeconds = (length / (frameSize * frameRate)); + duration = Math.round(durationInSeconds * 1000); + logger.debug("Duration of input stream : {}", duration); + } + return audioInputStream; + } else { + throw new UnsupportedAudioFormatException("Pulseaudio audio sink can only play pcm or mp3 stream", + audioFormat); + } + } + + /** + * This class add reset capability (on the first bytes only) + * to an AudioStream. This is necessary for the parsing / format detection. + * + */ + public static class ResetableInputStream extends InputStream { + + private static final int BUFFER_LENGTH = 10000; + + private final InputStream originalInputStream; + + private int position = -1; + private int markPosition = -1; + private int maxPreviousPosition = -2; + + private byte[] startingBuffer = new byte[BUFFER_LENGTH + 1]; + + public ResetableInputStream(InputStream originalInputStream) { + this.originalInputStream = originalInputStream; + } + + @Override + public void close() throws IOException { + originalInputStream.close(); + } + + @Override + public int read() throws IOException { + if (position >= BUFFER_LENGTH || originalInputStream.markSupported()) { + return originalInputStream.read(); + } else { + position++; + if (position <= maxPreviousPosition) { + return Byte.toUnsignedInt(startingBuffer[position]); + } else { + int currentByte = originalInputStream.read(); + startingBuffer[position] = (byte) currentByte; + maxPreviousPosition = position; + return currentByte; + } + } + } + + @Override + public synchronized void mark(int readlimit) { + if (originalInputStream.markSupported()) { + originalInputStream.mark(readlimit); + } + markPosition = position; + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public synchronized void reset() throws IOException { + if (originalInputStream.markSupported()) { + originalInputStream.reset(); + } else if (position >= BUFFER_LENGTH) { + throw new IOException("mark/reset not supported above " + BUFFER_LENGTH + " bytes"); + } + position = markPosition; + } + } +} diff --git a/bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/PulseAudioAudioSink.java b/bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/PulseAudioAudioSink.java index 9de4d3cc7ccc7..dee60aaf9a134 100644 --- a/bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/PulseAudioAudioSink.java +++ b/bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/PulseAudioAudioSink.java @@ -13,23 +13,16 @@ package org.openhab.binding.pulseaudio.internal; import java.io.IOException; -import java.io.InputStream; import java.net.Socket; import java.time.Duration; import java.time.Instant; import java.util.HashSet; import java.util.Locale; -import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import javazoom.spi.mpeg.sampled.convert.MpegFormatConversionProvider; -import javazoom.spi.mpeg.sampled.file.MpegAudioFileReader; -import javax.sound.sampled.AudioFileFormat; -import javax.sound.sampled.AudioInputStream; -import javax.sound.sampled.AudioSystem; import javax.sound.sampled.UnsupportedAudioFileException; import org.eclipse.jdt.annotation.NonNullByDefault; @@ -44,7 +37,6 @@ import org.openhab.core.library.types.PercentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.tritonus.share.sampled.file.TAudioFileFormat; /** * The audio sink for openhab, implemented by a connection to a pulseaudio sink @@ -90,52 +82,6 @@ public String getId() { return pulseaudioHandler.getThing().getLabel(); } - /** - * Convert MP3 to PCM, as this is the only possible format - * - * @param input - * @return - */ - private @Nullable AudioStreamAndDuration getPCMStreamFromMp3Stream(InputStream input) { - try { - - MpegAudioFileReader mpegAudioFileReader = new MpegAudioFileReader(); - - int duration = -1; - if (input instanceof FixedLengthAudioStream) { - final Long audioFileLength = ((FixedLengthAudioStream) input).length(); - AudioFileFormat audioFileFormat = mpegAudioFileReader.getAudioFileFormat(input); - if (audioFileFormat instanceof TAudioFileFormat) { - Map taudioFileFormatProperties = ((TAudioFileFormat) audioFileFormat).properties(); - if (taudioFileFormatProperties.containsKey("mp3.framesize.bytes") - && taudioFileFormatProperties.containsKey("mp3.framerate.fps")) { - Integer frameSize = (Integer) taudioFileFormatProperties.get("mp3.framesize.bytes"); - Float frameRate = (Float) taudioFileFormatProperties.get("mp3.framerate.fps"); - if (frameSize != null && frameRate != null) { - duration = Math.round((audioFileLength / (frameSize * frameRate)) * 1000); - } - } - } - input.reset(); - } - - AudioInputStream sourceAIS = mpegAudioFileReader.getAudioInputStream(input); - javax.sound.sampled.AudioFormat sourceFormat = sourceAIS.getFormat(); - - MpegFormatConversionProvider mpegconverter = new MpegFormatConversionProvider(); - javax.sound.sampled.AudioFormat convertFormat = new javax.sound.sampled.AudioFormat( - javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED, sourceFormat.getSampleRate(), 16, - sourceFormat.getChannels(), sourceFormat.getChannels() * 2, sourceFormat.getSampleRate(), false); - - AudioInputStream audioInputStreamConverted = mpegconverter.getAudioInputStream(convertFormat, sourceAIS); - return new AudioStreamAndDuration(audioInputStreamConverted, duration); - - } catch (IOException | UnsupportedAudioFileException e) { - logger.warn("Cannot convert this mp3 stream to pcm stream: {}", e.getMessage()); - } - return null; - } - /** * Connect to pulseaudio with the simple protocol * @@ -168,23 +114,6 @@ public void disconnect() { } } - private AudioStreamAndDuration getWavAudioAndDuration(AudioStream audioStream) { - int duration = -1; - if (audioStream instanceof FixedLengthAudioStream) { - final Long audioFileLength = ((FixedLengthAudioStream) audioStream).length(); - try { - AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(audioStream); - int frameSize = audioInputStream.getFormat().getFrameSize(); - float frameRate = audioInputStream.getFormat().getFrameRate(); - float durationInSeconds = (audioFileLength / (frameSize * frameRate)); - duration = Math.round(durationInSeconds * 1000); - } catch (UnsupportedAudioFileException | IOException e) { - logger.warn("Error when getting duration information from AudioFile"); - } - } - return new AudioStreamAndDuration(audioStream, duration); - } - @Override public void process(@Nullable AudioStream audioStream) throws UnsupportedAudioFormatException, UnsupportedAudioStreamException { @@ -193,34 +122,22 @@ public void process(@Nullable AudioStream audioStream) return; } - AudioStreamAndDuration audioInputStreamAndDuration = null; - try { - - if (AudioFormat.MP3.isCompatible(audioStream.getFormat())) { - audioInputStreamAndDuration = getPCMStreamFromMp3Stream(audioStream); - } else if (AudioFormat.WAV.isCompatible(audioStream.getFormat())) { - audioInputStreamAndDuration = getWavAudioAndDuration(audioStream); - } else { - throw new UnsupportedAudioFormatException("pulseaudio audio sink can only play pcm or mp3 stream", - audioStream.getFormat()); - } - + try (ConvertedInputStream normalizedPCMStream = new ConvertedInputStream(audioStream)) { for (int countAttempt = 1; countAttempt <= 2; countAttempt++) { // two attempts allowed try { connectIfNeeded(); final Socket clientSocketLocal = clientSocket; - if (audioInputStreamAndDuration != null && clientSocketLocal != null) { + if (clientSocketLocal != null) { // send raw audio to the socket and to pulse audio isIdle = false; Instant start = Instant.now(); - audioInputStreamAndDuration.inputStream.transferTo(clientSocketLocal.getOutputStream()); - if (audioInputStreamAndDuration.duration != -1) { // ensure, if the sound has a duration + normalizedPCMStream.transferTo(clientSocketLocal.getOutputStream()); + if (normalizedPCMStream.getDuration() != -1) { // ensure, if the sound has a duration // that we let at least this time for the system to play Instant end = Instant.now(); long millisSecondTimedToSendAudioData = Duration.between(start, end).toMillis(); - if (millisSecondTimedToSendAudioData < audioInputStreamAndDuration.duration) { - long timeToSleep = audioInputStreamAndDuration.duration - - millisSecondTimedToSendAudioData; + if (millisSecondTimedToSendAudioData < normalizedPCMStream.getDuration()) { + long timeToSleep = normalizedPCMStream.getDuration() - millisSecondTimedToSendAudioData; logger.debug("Sleep time to let the system play sound : {}", timeToSleep); Thread.sleep(timeToSleep); } @@ -243,15 +160,11 @@ public void process(@Nullable AudioStream audioStream) break; } } + } catch (UnsupportedAudioFileException | IOException e) { + throw new UnsupportedAudioFormatException("Cannot send sound to the pulseaudio sink", + audioStream.getFormat(), e); } finally { - try { - if (audioInputStreamAndDuration != null) { - audioInputStreamAndDuration.inputStream.close(); - } - audioStream.close(); - scheduleDisconnect(); - } catch (IOException e) { - } + scheduleDisconnect(); } isIdle = true; } @@ -286,15 +199,4 @@ public PercentType getVolume() { public void setVolume(PercentType volume) { pulseaudioHandler.setVolume(volume.intValue()); } - - private static class AudioStreamAndDuration { - private InputStream inputStream; - private int duration; - - public AudioStreamAndDuration(InputStream inputStream, int duration) { - super(); - this.inputStream = inputStream; - this.duration = duration + 200; // introduce some delay - } - } }