[audio] Enhance AudioSink capabilities using the AudioServlet (#3461)

* [audio] More capabilities for AudioSink using the AudioServlet

AudioServlet can now serve all type of AudioStream multiple times by buffering data in memory or in temporary file.
Adding method to ease disposal of temporary file after playing a sound
Adding an identifyier to audio stream for further development (allow audio sink to cache computation data)

We can now send audio with a Runnable for a delayed task to be executed after. This delayed task includes temporary file deletion and volume restoration.
This is a no breaking change / no behaviour modification for other addon AudioSink, as existing AudioSink must explicitly override the old behaviour to use this capability.
Add AudioSinkSync / AudioSinkAsync abstract classes to use this capability easily.
WebAudioSink now implements this capability, with the help of a modified AudioServlet

Adding (approximative, better than nothing) sound duration computation method for MP3 and WAV.
Use this sound duration computation to guess when the async sound is finished and when to do the post process (i.e. volume restoration)

Signed-off-by: Gwendal Roulleau <gwendal.roulleau@gmail.com>

bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioHTTPServer.java

@@ -12,6 +12,9 @@
  */
 package org.openhab.core.audio;
 
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+
 import org.eclipse.jdt.annotation.NonNullByDefault;
 import org.openhab.core.audio.internal.AudioServlet;
 
@@ -34,19 +37,48 @@ public interface AudioHTTPServer {
  *
  * @param stream the stream to serve on HTTP
  * @return the relative URL to access the stream starting with a '/'
+ * @deprecated Use {@link AudioHTTPServer#serve(AudioStream, int, boolean, CompletableFuture)}
  */
+ @Deprecated
  String serve(AudioStream stream);
 
  /**
  * Creates a relative url for a given {@link AudioStream} where it can be requested multiple times within the given
  * time frame.
- * This method only accepts {@link FixedLengthAudioStream}s, since it needs to be able to create multiple concurrent
- * streams from it, which isn't possible with a regular {@link AudioStream}.
+ * This method accepts all {@link AudioStream}s, but it is better to use {@link ClonableAudioStream}s. If generic
+ * {@link AudioStream} is used, the method tries to add the Clonable capability by storing it in a small memory
+ * buffer, e.g {@link ByteArrayAudioStream}, or in a cached file if the stream reached the buffer capacity,
+ * or fails if the stream is too long.
  * Streams are closed, once they expire.
  *
  * @param stream the stream to serve on HTTP
  * @param seconds number of seconds for which the stream is available through HTTP
  * @return the relative URL to access the stream starting with a '/'
+ * @deprecated Use {@link AudioHTTPServer#serve(AudioStream, int, boolean, CompletableFuture)}
+ */
+ @Deprecated
+ String serve(AudioStream stream, int seconds);
+
+ /**
+ * Creates a relative url for a given {@link AudioStream} where it can be requested one or multiple times within the
+ * given time frame.
+ * This method accepts all {@link AudioStream}s, but if multiTimeStream is set to true it is better to use
+ * {@link ClonableAudioStream}s. Otherwise, if a generic {@link AudioStream} is used, the method will then try
+ * to add the Clonable capability by storing it in a small memory buffer, e.g {@link ByteArrayAudioStream}, or in a
+ * cached file if the stream reached the buffer capacity, or fails to render the sound completely if the stream is
+ * too long.
+ * A {@link CompletableFuture} is used to inform the caller that the playback ends in order to clean
+ * resources and run delayed task, such as restoring volume.
+ * Streams are closed, once they expire.
+ *
+ * @param stream the stream to serve on HTTP
+ * @param seconds number of seconds for which the stream is available through HTTP. The stream will be deleted only
+ * if not started, so you can set a duration shorter than the track's duration.
+ * @param multiTimeStream set to true if this stream should be played multiple time, and thus needs to be made
+ * Cloneable if it is not already.
+ * @return information about the {@link StreamServed}, including the relative URL to access the stream starting with
+ * a '/', and a CompletableFuture to know when the playback ends.
+ * @throws IOException when the stream is not a {@link ClonableAudioStream} and we cannot get or store it on disk.
  */
- String serve(FixedLengthAudioStream stream, int seconds);
+ StreamServed serve(AudioStream stream, int seconds, boolean multiTimeStream) throws IOException;
 }

bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioManager.java

@@ -252,4 +252,15 @@ public interface AudioManager {
  * @return ids of matching sinks
  */
  Set<String> getSinkIds(String pattern);
+
+ /**
+ * Handles a volume command change and returns a Runnable to restore it.
+ * Returning a Runnable allows us to have a no-op Runnable if changing volume back is not needed, and conveniently
+ * keeping it as one liner usable in a chain for the caller.
+ *
+ * @param volume The volume to set
+ * @param sink The sink to set the volume to
+ * @return A runnable to restore the volume to its previous value, or no-operation if no change is required.
+ */
+ Runnable handleVolumeCommand(@Nullable PercentType volume, AudioSink sink);
 }

bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioSink.java

@@ -15,6 +15,7 @@
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 
 import org.eclipse.jdt.annotation.NonNullByDefault;
 import org.eclipse.jdt.annotation.Nullable;
@@ -58,13 +59,47 @@ public interface AudioSink {
  *
  * In case the audioStream is null, this should be interpreted as a request to end any currently playing stream.
  *
+ * When the stream is not needed anymore, if the stream implements the {@link org.openhab.core.common.Disposable}
+ * interface, the sink should hereafter get rid of it by calling the dispose method.
+ *
  * @param audioStream the audio stream to play or null to keep quiet
  * @throws UnsupportedAudioFormatException If audioStream format is not supported
  * @throws UnsupportedAudioStreamException If audioStream is not supported
+ * @deprecated Use {@link AudioSink#processAndComplete(AudioStream)}
  */
+ @Deprecated
  void process(@Nullable AudioStream audioStream)
  throws UnsupportedAudioFormatException, UnsupportedAudioStreamException;
 
+ /**
+ * Processes the passed {@link AudioStream}, and returns a CompletableFuture that should complete when the sound is
+ * fully played. It is the sink responsibility to complete this future.
+ *
+ * If the passed {@link AudioStream} is not supported by this instance, an {@link UnsupportedAudioStreamException}
+ * is thrown.
+ *
+ * If the passed {@link AudioStream} has an {@link AudioFormat} not supported by this instance,
+ * an {@link UnsupportedAudioFormatException} is thrown.
+ *
+ * In case the audioStream is null, this should be interpreted as a request to end any currently playing stream.
+ *
+ * When the stream is not needed anymore, if the stream implements the {@link org.openhab.core.common.Disposable}
+ * interface, the sink should hereafter get rid of it by calling the dispose method.
+ *
+ * @param audioStream the audio stream to play or null to keep quiet
+ * @return A future completed when the sound is fully played. The method can instead complete with
+ * UnsupportedAudioFormatException if the audioStream format is not supported, or
+ * UnsupportedAudioStreamException If audioStream is not supported
+ */
+ default CompletableFuture<@Nullable Void> processAndComplete(@Nullable AudioStream audioStream) {
+ try {
+ process(audioStream);
+ } catch (UnsupportedAudioFormatException | UnsupportedAudioStreamException e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ return CompletableFuture.completedFuture(null);
+ }
+
  /**
  * Gets a set containing all supported audio formats
  *

bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioSinkAsync.java

@@ -0,0 +1,113 @@
+/**
+ * Copyright (c) 2010-2023 Contributors to the openHAB project
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.openhab.core.audio;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+import org.eclipse.jdt.annotation.NonNullByDefault;
+import org.eclipse.jdt.annotation.Nullable;
+import org.openhab.core.common.Disposable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Definition of an audio output like headphones, a speaker or for writing to
+ * a file / clip.
+ * Helper class for asynchronous sink : when the process() method returns, the {@link AudioStream}
+ * may or may not be played. It is the responsibility of the implementing AudioSink class to
+ * complete the CompletableFuture when playing is done. Any delayed tasks will then be performed, such as volume
+ * restoration.
+ *
+ * @author Gwendal Roulleau - Initial contribution
+ */
+@NonNullByDefault
+public abstract class AudioSinkAsync implements AudioSink {
+
+ private final Logger logger = LoggerFactory.getLogger(AudioSinkAsync.class);
+
+ protected final Map<AudioStream, CompletableFuture<@Nullable Void>> runnableByAudioStream = new HashMap<>();
+
+ @Override
+ public CompletableFuture<@Nullable Void> processAndComplete(@Nullable AudioStream audioStream) {
+ CompletableFuture<@Nullable Void> completableFuture = new CompletableFuture<@Nullable Void>();
+ if (audioStream != null) {
+ runnableByAudioStream.put(audioStream, completableFuture);
+ }
+ try {
+ processAsynchronously(audioStream);
+ } catch (UnsupportedAudioFormatException | UnsupportedAudioStreamException e) {
+ completableFuture.completeExceptionally(e);
+ }
+ if (audioStream == null) {
+ // No need to delay the post process task
+ completableFuture.complete(null);
+ }
+ return completableFuture;
+ }
+
+ @Override
+ public void process(@Nullable AudioStream audioStream)
+ throws UnsupportedAudioFormatException, UnsupportedAudioStreamException {
+ processAsynchronously(audioStream);
+ }
+
+ /**
+ * Processes the passed {@link AudioStream} asynchronously. This method is expected to return before the stream is
+ * fully played. This is the sink responsibility to call the {@link AudioSinkAsync#playbackFinished(AudioStream)}
+ * when it is.
+ *
+ * If the passed {@link AudioStream} is not supported by this instance, an {@link UnsupportedAudioStreamException}
+ * is thrown.
+ *
+ * If the passed {@link AudioStream} has an {@link AudioFormat} not supported by this instance,
+ * an {@link UnsupportedAudioFormatException} is thrown.
+ *
+ * In case the audioStream is null, this should be interpreted as a request to end any currently playing stream.
+ *
+ * @param audioStream the audio stream to play or null to keep quiet
+ * @throws UnsupportedAudioFormatException If audioStream format is not supported
+ * @throws UnsupportedAudioStreamException If audioStream is not supported
+ */
+ protected abstract void processAsynchronously(@Nullable AudioStream audioStream)
+ throws UnsupportedAudioFormatException, UnsupportedAudioStreamException;
+
+ /**
+ * Will complete the future previously returned, allowing the core to run delayed task.
+ *
+ * @param audioStream The AudioStream is the key to find the delayed CompletableFuture in the storage.
+ */
+ protected void playbackFinished(AudioStream audioStream) {
+ CompletableFuture<@Nullable Void> completableFuture = runnableByAudioStream.remove(audioStream);
+ if (completableFuture != null) {
+ completableFuture.complete(null);
+ }
+
+ // if the stream is not needed anymore, then we should call back the AudioStream to let it a chance
+ // to auto dispose.
+ if (audioStream instanceof Disposable disposableAudioStream) {
+ try {
+ disposableAudioStream.dispose();
+ } catch (IOException e) {
+ String fileName = audioStream instanceof FileAudioStream file ? file.toString() : "unknown";
+ if (logger.isDebugEnabled()) {
+ logger.debug("Cannot dispose of stream {}", fileName, e);
+ } else {
+ logger.warn("Cannot dispose of stream {}, reason {}", fileName, e.getMessage());
+ }
+ }
+ }
+ }
+}

bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioSinkSync.java

@@ -0,0 +1,85 @@
+/**
+ * Copyright (c) 2010-2023 Contributors to the openHAB project
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.openhab.core.audio;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+
+import org.eclipse.jdt.annotation.NonNullByDefault;
+import org.eclipse.jdt.annotation.Nullable;
+import org.openhab.core.common.Disposable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Definition of an audio output like headphones, a speaker or for writing to
+ * a file / clip.
+ * Helper class for synchronous sink : when the process() method returns,
+ * the source is considered played, and could be disposed.
+ * Any delayed tasks can then be performed, such as volume restoration.
+ *
+ * @author Gwendal Roulleau - Initial contribution
+ */
+@NonNullByDefault
+public abstract class AudioSinkSync implements AudioSink {
+
+ private final Logger logger = LoggerFactory.getLogger(AudioSinkSync.class);
+
+ @Override
+ public CompletableFuture<@Nullable Void> processAndComplete(@Nullable AudioStream audioStream) {
+ try {
+ processSynchronously(audioStream);
+ return CompletableFuture.completedFuture(null);
+ } catch (UnsupportedAudioFormatException | UnsupportedAudioStreamException e) {
+ return CompletableFuture.failedFuture(e);
+ } finally {
+ // as the stream is not needed anymore, we should dispose of it
+ if (audioStream instanceof Disposable disposableAudioStream) {
+ try {
+ disposableAudioStream.dispose();
+ } catch (IOException e) {
+ String fileName = audioStream instanceof FileAudioStream file ? file.toString() : "unknown";
+ if (logger.isDebugEnabled()) {
+ logger.debug("Cannot dispose of stream {}", fileName, e);
+ } else {
+ logger.warn("Cannot dispose of stream {}, reason {}", fileName, e.getMessage());
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void process(@Nullable AudioStream audioStream)
+ throws UnsupportedAudioFormatException, UnsupportedAudioStreamException {
+ processSynchronously(audioStream);
+ }
+
+ /**
+ * Processes the passed {@link AudioStream} and returns only when the playback is ended.
+ *
+ * If the passed {@link AudioStream} is not supported by this instance, an {@link UnsupportedAudioStreamException}
+ * is thrown.
+ *
+ * If the passed {@link AudioStream} has an {@link AudioFormat} not supported by this instance,
+ * an {@link UnsupportedAudioFormatException} is thrown.
+ *
+ * In case the audioStream is null, this should be interpreted as a request to end any currently playing stream.
+ *
+ * @param audioStream the audio stream to play or null to keep quiet
+ * @throws UnsupportedAudioFormatException If audioStream format is not supported
+ * @throws UnsupportedAudioStreamException If audioStream is not supported
+ */
+ protected abstract void processSynchronously(@Nullable AudioStream audioStream)
+ throws UnsupportedAudioFormatException, UnsupportedAudioStreamException;
+}

bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioStream.java

@@ -15,6 +15,7 @@
 import java.io.InputStream;
 
 import org.eclipse.jdt.annotation.NonNullByDefault;
+import org.eclipse.jdt.annotation.Nullable;
 
 /**
  * Wrapper for a source of audio data.
@@ -37,4 +38,14 @@ public abstract class AudioStream extends InputStream {
  * @return The supported audio format
  */
  public abstract AudioFormat getFormat();
+
+ /**
+ * Usefull for sinks playing the same stream multiple times,
+ * to avoid already done computation (like reencoding).
+ *
+ * @return A string uniquely identifying the stream.
+ */
+ public @Nullable String getId() {
+ return null;
+ }
 }

bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/ClonableAudioStream.java

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) 2010-2023 Contributors to the openHAB project
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.openhab.core.audio;
+
+import java.io.InputStream;
+
+import org.eclipse.jdt.annotation.NonNullByDefault;
+
+/**
+ * This is an {@link AudioStream}, that can be cloned
+ *
+ * @author Gwendal Roulleau - Initial contribution, separation from FixedLengthAudioStream
+ */
+@NonNullByDefault
+public abstract class ClonableAudioStream extends AudioStream {
+
+ /**
+ * Returns a new, fully independent stream instance, which can be read and closed without impacting the original
+ * instance.
+ *
+ * @return a new input stream that can be consumed by the caller
+ * @throws AudioException if stream cannot be created
+ */
+ public abstract InputStream getClonedStream() throws AudioException;
+}