Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Correct JavaFxPlayVideoAndAudio sample to synchronize audio and video… #1662

Merged
merged 5 commits into from
Jun 21, 2021
Merged

Correct JavaFxPlayVideoAndAudio sample to synchronize audio and video… #1662

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dc7c2a5
Correct JavaFxPlayVideoAndAudio sample to synchronize audio and vodeo…
jpsacha Jun 13, 2021
6ea8dac
Remove obsolete comment
jpsacha Jun 13, 2021
ae365b8
Correction from PR comments and improvements to quality of playback
jpsacha Jun 16, 2021
9595a22
Use Thread.currentThread().interrupt() after InterruptedException
jpsacha Jun 17, 2021
d68676e
Update CHANGELOG.md
saudet Jun 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@

* Enhance audio and video synchronization of `JavaFxPlayVideoAndAudio` sample ([pull #1662](https://github.com/bytedeco/javacv/pull/1662))
* Add `FrameGrabber.grabAtFrameRate()` to simulate a device or stream when reading from files ([pull #1659](https://github.com/bytedeco/javacv/pull/1659))
* Update `FFmpegFrameGrabber` and `FFmpegFrameRecorder` with new `avcodec` API ([issue #1498](https://github.com/bytedeco/javacv/issues/1498))
* Add new `Similarity` sample with PSNR and MSSIM ([pull #1622](https://github.com/bytedeco/javacv/pull/1622))
Expand Down
132 changes: 104 additions & 28 deletions samples/JavaFxPlayVideoAndAudio.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import java.util.logging.Logger;
import javafx.application.Application;
import javafx.application.Platform;
import javafx.embed.swing.SwingFXUtils;
import javafx.scene.Scene;
import javafx.scene.image.Image;
import javafx.scene.image.ImageView;
Expand All @@ -19,13 +18,44 @@
import javax.sound.sampled.SourceDataLine;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameConverter;
import org.bytedeco.javacv.JavaFXFrameConverter;

/**
* @author Dmitriy Gerashenko <d.a.gerashenko@gmail.com>
* @author Jarek Sacha
*/
public class JavaFxPlayVideoAndAudio extends Application {

private static class PlaybackTimer {
private Long startTime = -1L;
private final DataLine soundLine;

public PlaybackTimer(DataLine soundLine) {
this.soundLine = soundLine;
}

public PlaybackTimer() {
this.soundLine = null;
}

public void start() {
if (soundLine == null) {
startTime = System.nanoTime();
}
}

public long elapsedMicros() {
if (soundLine == null) {
if (startTime < 0) {
throw new IllegalStateException("PlaybackTimer not initialized.");
}
return (System.nanoTime() - startTime) / 1000;
} else {
return soundLine.getMicrosecondPosition();
}
}
}

private static final Logger LOG = Logger.getLogger(JavaFxPlayVideoAndAudio.class.getName());

private static volatile Thread playThread;
Expand Down Expand Up @@ -56,28 +86,65 @@ public void start(final Stage primaryStage) throws Exception {
grabber.start();
primaryStage.setWidth(grabber.getImageWidth());
primaryStage.setHeight(grabber.getImageHeight());
final AudioFormat audioFormat = new AudioFormat(grabber.getSampleRate(), 16, grabber.getAudioChannels(), true, true);
final PlaybackTimer playbackTimer;
final SourceDataLine soundLine;
if (grabber.getAudioChannels() > 0) {
final AudioFormat audioFormat = new AudioFormat(grabber.getSampleRate(), 16, grabber.getAudioChannels(), true, true);

final DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat);
soundLine = (SourceDataLine) AudioSystem.getLine(info);
soundLine.open(audioFormat);
soundLine.start();
playbackTimer = new PlaybackTimer(soundLine);
} else {
soundLine = null;
playbackTimer = new PlaybackTimer();
}

final DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat);
final SourceDataLine soundLine = (SourceDataLine) AudioSystem.getLine(info);
soundLine.open(audioFormat);
soundLine.start();
final JavaFXFrameConverter converter = new JavaFXFrameConverter();

final Java2DFrameConverter converter = new Java2DFrameConverter();
final ExecutorService audioExecutor = Executors.newSingleThreadExecutor();
final ExecutorService imageExecutor = Executors.newSingleThreadExecutor();

ExecutorService executor = Executors.newSingleThreadExecutor();
final long maxReadAheadBufferMicros = 1000 * 1000L;

long lastTimeStamp = -1L;
while (!Thread.interrupted()) {
Frame frame = grabber.grab();
final Frame frame = grabber.grab();
if (frame == null) {
break;
}
if (lastTimeStamp < 0) {
playbackTimer.start();
}
lastTimeStamp = frame.timestamp;
if (frame.image != null) {
final Image image = SwingFXUtils.toFXImage(converter.convert(frame), null);
Platform.runLater(new Runnable() { public void run() {
imageView.setImage(image);
}});
final Frame imageFrame = frame.clone();

imageExecutor.submit(new Runnable() {
public void run() {
final Image image = converter.convert(imageFrame);
final long timeStampDeltaMicros = imageFrame.timestamp - playbackTimer.elapsedMicros();
imageFrame.close();
if (timeStampDeltaMicros > 0) {
final long delayMillis = timeStampDeltaMicros / 1000L;
try {
Thread.sleep(delayMillis);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
Platform.runLater(new Runnable() {
public void run() {
imageView.setImage(image);
}
});
}
});
} else if (frame.samples != null) {
if (soundLine == null) {
throw new IllegalStateException("Internal error: sound playback not initialized");
}
final ShortBuffer channelSamplesShortBuffer = (ShortBuffer) frame.samples[0];
channelSamplesShortBuffer.rewind();

Expand All @@ -88,25 +155,34 @@ public void start(final Stage primaryStage) throws Exception {
outBuffer.putShort(val);
}

/**
* We need this because soundLine.write ignores
* interruptions during writing.
*/
try {
executor.submit(new Runnable() { public void run() {
audioExecutor.submit(new Runnable() {
public void run() {
soundLine.write(outBuffer.array(), 0, outBuffer.capacity());
outBuffer.clear();
}}).get();
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
}
}
});
}
final long timeStampDeltaMicros = frame.timestamp - playbackTimer.elapsedMicros();
if (timeStampDeltaMicros > maxReadAheadBufferMicros) {
Thread.sleep((timeStampDeltaMicros - maxReadAheadBufferMicros) / 1000);
}
}
executor.shutdownNow();
executor.awaitTermination(10, TimeUnit.SECONDS);
soundLine.stop();

if (!Thread.interrupted()) {
long delay = (lastTimeStamp - playbackTimer.elapsedMicros()) / 1000 +
Math.round(1 / grabber.getFrameRate() * 1000);
Thread.sleep(Math.max(0, delay));
}
grabber.stop();
grabber.release();
if (soundLine != null) {
soundLine.stop();
}
audioExecutor.shutdownNow();
audioExecutor.awaitTermination(10, TimeUnit.SECONDS);
imageExecutor.shutdownNow();
imageExecutor.awaitTermination(10, TimeUnit.SECONDS);

Platform.exit();
} catch (Exception exception) {
LOG.log(Level.SEVERE, null, exception);
Expand All @@ -117,7 +193,7 @@ public void start(final Stage primaryStage) throws Exception {
}

@Override
public void stop() throws Exception {
public void stop() {
playThread.interrupt();
}

Expand Down