diff --git a/app/meson.build b/app/meson.build index 060dc8db2f..cb2de6e9f6 100644 --- a/app/meson.build +++ b/app/meson.build @@ -322,10 +322,6 @@ if get_option('buildtype') == 'debug' 'src/util/str.c', 'src/util/strbuf.c', ]], - ['test_strutil', [ - 'tests/test_strutil.c', - 'src/util/str_util.c', - ]], ['test_vector', [ 'tests/test_vector.c', ]], diff --git a/app/src/control_msg.c b/app/src/control_msg.c index a57d8cc289..50b537c06f 100644 --- a/app/src/control_msg.c +++ b/app/src/control_msg.c @@ -141,6 +141,7 @@ sc_control_msg_serialize(const struct sc_control_msg *msg, unsigned char *buf) { case SC_CONTROL_MSG_TYPE_EXPAND_SETTINGS_PANEL: case SC_CONTROL_MSG_TYPE_COLLAPSE_PANELS: case SC_CONTROL_MSG_TYPE_ROTATE_DEVICE: + case SC_CONTROL_MSG_TYPE_REQUEST_KEY_FRAME: // no additional data return 1; default: @@ -229,6 +230,9 @@ sc_control_msg_log(const struct sc_control_msg *msg) { case SC_CONTROL_MSG_TYPE_ROTATE_DEVICE: LOG_CMSG("rotate device"); break; + case SC_CONTROL_MSG_TYPE_REQUEST_KEY_FRAME: + LOG_CMSG("request key frame"); + break; default: LOG_CMSG("unknown type: %u", (unsigned) msg->type); break; diff --git a/app/src/control_msg.h b/app/src/control_msg.h index 1463fddc24..b3b0ad1ed8 100644 --- a/app/src/control_msg.h +++ b/app/src/control_msg.h @@ -33,6 +33,7 @@ enum sc_control_msg_type { SC_CONTROL_MSG_TYPE_SET_CLIPBOARD, SC_CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE, SC_CONTROL_MSG_TYPE_ROTATE_DEVICE, + SC_CONTROL_MSG_TYPE_REQUEST_KEY_FRAME, }; enum sc_screen_power_mode { diff --git a/app/src/forward.c b/app/src/forward.c index 3db5666806..5d1fc3d265 100644 --- a/app/src/forward.c +++ b/app/src/forward.c @@ -29,6 +29,8 @@ #include +#include "control_msg.h" +#include "controller.h" #include "events.h" #include "packet.h" #include "util/log.h" @@ -99,6 +101,7 @@ bool forward_init(struct forward *forward, const char *socketname) { forward->socket = socket(PF_LOCAL, SOCK_STREAM, 0); if (forward->socket < 0) { LOGE("Could not create forward socket"); + sc_cond_destroy(&forward->queue_cond); sc_mutex_destroy(&forward->mutex); return false; } @@ -115,6 +118,7 @@ bool forward_init(struct forward *forward, const char *socketname) { if(0 != err) { LOGE("Could not bind to forward socket [%d][%s]", errno, strerror(errno)); close(forward->socket); + sc_cond_destroy(&forward->queue_cond); sc_mutex_destroy(&forward->mutex); return false; } @@ -123,6 +127,7 @@ bool forward_init(struct forward *forward, const char *socketname) { if (!ok) { LOGE("Could not create receiver_mutex"); close(forward->socket); + sc_cond_destroy(&forward->queue_cond); sc_mutex_destroy(&forward->mutex); return false; } @@ -140,6 +145,9 @@ static void notify_disconnected() { void forward_destroy(struct forward *forward) { close(forward->socket); forward->socket = -1; + sc_mutex_destroy(&forward->receiver_mutex); + sc_cond_destroy(&forward->queue_cond); + sc_mutex_destroy(&forward->mutex); } static bool send_packet(struct forward *forward, const AVPacket *packet) { @@ -309,6 +317,16 @@ run_forward_server(void *data) { if (size>=4 && 0==strncmp("STOP", request, 4)) { should_stop = true; } + if (size>=3 && 0==strncmp("KEY", request, 3)) { + struct sc_control_msg msg = { + .type = SC_CONTROL_MSG_TYPE_REQUEST_KEY_FRAME, + }; + if (forward->controller) { + if (!sc_controller_push_msg(forward->controller, &msg)) { + LOGW("Could not request key frame"); + } + } + } } } } @@ -333,7 +351,8 @@ run_forward(void *data) { sc_mutex_lock(&forward->mutex); while (!forward->stopped && sc_queue_is_empty(&forward->queue)) { - sc_cond_timedwait(&forward->queue_cond, &forward->mutex, 1000); + sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(1000); + sc_cond_timedwait(&forward->queue_cond, &forward->mutex, deadline); } if (forward->stopped) { @@ -427,12 +446,14 @@ forward_push(struct forward *forward, const AVPacket *packet) { if (forward->stopped) { // reject any new packet (this will stop the stream) + sc_mutex_unlock(&forward->mutex); return false; } struct forward_packet *rec = forward_packet_new(packet); if (!rec) { LOGE("Could not allocate forward packet"); + sc_mutex_unlock(&forward->mutex); return false; } diff --git a/app/src/forward.h b/app/src/forward.h index 82ea87fd11..0dc6eb1045 100644 --- a/app/src/forward.h +++ b/app/src/forward.h @@ -40,6 +40,10 @@ struct forward { sc_cond queue_cond; bool stopped; struct forward_queue queue; + + // NOTE(Frankleonrose): Assigned after controller is initialized, before + // packets start flowing. + struct sc_controller *controller; }; bool forward_init(struct forward *forward, const char *socketname); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 45bc45faf5..a965508642 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -615,6 +615,9 @@ scrcpy(struct scrcpy_options *options) { } } + if (fwd) { + fwd->controller = controller; + } } // There is a controller if and only if control is enabled diff --git a/app/tests/test_control_msg_serialize.c b/app/tests/test_control_msg_serialize.c index 72e3d87a8d..9ec97d1ef8 100644 --- a/app/tests/test_control_msg_serialize.c +++ b/app/tests/test_control_msg_serialize.c @@ -320,6 +320,21 @@ static void test_serialize_rotate_device(void) { assert(!memcmp(buf, expected, sizeof(expected))); } +static void test_serialize_request_key_frame(void) { + struct sc_control_msg msg = { + .type = SC_CONTROL_MSG_TYPE_REQUEST_KEY_FRAME, + }; + + unsigned char buf[SC_CONTROL_MSG_MAX_SIZE]; + size_t size = sc_control_msg_serialize(&msg, buf); + assert(size == 1); + + const unsigned char expected[] = { + SC_CONTROL_MSG_TYPE_REQUEST_KEY_FRAME, + }; + assert(!memcmp(buf, expected, sizeof(expected))); +} + int main(int argc, char *argv[]) { (void) argc; (void) argv; @@ -338,5 +353,6 @@ int main(int argc, char *argv[]) { test_serialize_set_clipboard_long(); test_serialize_set_screen_power_mode(); test_serialize_rotate_device(); + test_serialize_request_key_frame(); return 0; } diff --git a/mobot.mk b/mobot.mk index c976852e20..d184601653 100644 --- a/mobot.mk +++ b/mobot.mk @@ -56,7 +56,8 @@ build-app: $(AVLIBS) LDFLAGS="-Wl,-lm -Wl,-lpthread $(MAC_LDFLAGS)" \ meson build-app --buildtype release --strip -Db_lto=true \ -Dlocal_libav=$(AVDIR) \ - -Dcompile_server=false \ + -Dcompile_server=true \ + -Dusb=false \ || (ret=$$?; rm -rf $@ && exit $$ret) scrcpy: build-app @@ -65,11 +66,10 @@ scrcpy: build-app test: rm -rf build_test CFLAGS="-DMOBOT_VERSION='\"$(GIT_DESCRIBE)/$(ARCH)\"'" \ - meson build_test --buildtype debug --strip -Db_sanitize=address -Db_lto=true -Dlocal_libav=build-libav -Dcompile_server=false + meson build_test --buildtype debug --strip -Db_sanitize=address -Db_lto=true -Dlocal_libav=build-libav -Dcompile_server=false -Dusb=false ninja -Cbuild_test build_test/app/test_cli build_test/app/test_control_msg_serialize - build_test/app/test_strutil build_test/app/test_buffer_util build_test/app/test_device_msg_deserialize build_test/app/test_cbuf diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java index 99eb805f28..20d60f36fd 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessage.java @@ -17,6 +17,7 @@ public final class ControlMessage { public static final int TYPE_SET_CLIPBOARD = 9; public static final int TYPE_SET_SCREEN_POWER_MODE = 10; public static final int TYPE_ROTATE_DEVICE = 11; + public static final int TYPE_REQUEST_KEY_FRAME = 12; public static final long SEQUENCE_INVALID = 0; diff --git a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java index 24dc5e50e1..cc202770a6 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java +++ b/server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java @@ -84,6 +84,7 @@ public ControlMessage next() { case ControlMessage.TYPE_EXPAND_SETTINGS_PANEL: case ControlMessage.TYPE_COLLAPSE_PANELS: case ControlMessage.TYPE_ROTATE_DEVICE: + case ControlMessage.TYPE_REQUEST_KEY_FRAME: msg = ControlMessage.createEmpty(type); break; default: diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 913371ee06..693538ecd9 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -133,6 +133,9 @@ private void handleEvent() throws IOException { } } break; + case ControlMessage.TYPE_REQUEST_KEY_FRAME: + device.requestKeyFrame(); + break; case ControlMessage.TYPE_ROTATE_DEVICE: Device.rotateDevice(); break; diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java index 763a7fadbd..94e6ed1a35 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/Device.java @@ -34,6 +34,10 @@ public final class Device { private static final ServiceManager SERVICE_MANAGER = new ServiceManager(); private static final Settings SETTINGS = new Settings(SERVICE_MANAGER); + public interface KeyFrameListener { + void onKeyFrameRequested(); + } + public interface RotationListener { void onRotationChanged(int rotation); } @@ -48,6 +52,7 @@ public interface ClipboardListener { private final int lockVideoOrientation; private ScreenInfo screenInfo; + private KeyFrameListener keyFrameListener; private RotationListener rotationListener; private ClipboardListener clipboardListener; private final AtomicBoolean isSettingClipboard = new AtomicBoolean(); @@ -223,6 +228,10 @@ public static boolean isScreenOn() { return SERVICE_MANAGER.getPowerManager().isScreenOn(); } + public synchronized void setKeyFrameListener(KeyFrameListener keyFrameListener) { + this.keyFrameListener = keyFrameListener; + } + public synchronized void setRotationListener(RotationListener rotationListener) { this.rotationListener = rotationListener; } @@ -295,6 +304,12 @@ public static boolean powerOffScreen(int displayId) { return pressReleaseKeycode(KeyEvent.KEYCODE_POWER, displayId, Device.INJECT_MODE_ASYNC); } + public void requestKeyFrame() { + if (keyFrameListener!=null) { + keyFrameListener.onKeyFrameRequested(); + } + } + /** * Disable auto-rotation (if enabled), set the screen rotation and re-enable auto-rotation (if it was enabled). */ diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index e95896d370..a31d8b7e4e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -2,13 +2,16 @@ import com.genymobile.scrcpy.wrappers.SurfaceControl; +import android.graphics.Canvas; import android.graphics.Rect; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.MediaFormat; import android.os.Build; +import android.os.Bundle; import android.os.IBinder; +import android.os.SystemClock; import android.view.Surface; import java.io.FileDescriptor; @@ -17,9 +20,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.atomic.AtomicBoolean; -public class ScreenEncoder implements Device.RotationListener { +public class ScreenEncoder implements Device.RotationListener, Device.KeyFrameListener { private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms @@ -31,6 +36,7 @@ public class ScreenEncoder implements Device.RotationListener { private static final long PACKET_FLAG_CONFIG = 1L << 63; private static final long PACKET_FLAG_KEY_FRAME = 1L << 62; + private final AtomicBoolean keyFrameRequested = new AtomicBoolean(); private final AtomicBoolean rotationChanged = new AtomicBoolean(); private final ByteBuffer headerBuffer = ByteBuffer.allocate(12); @@ -43,6 +49,7 @@ public class ScreenEncoder implements Device.RotationListener { private long ptsOrigin; private boolean firstFrameSent; + private long requestKeyFrameTime = 0; public ScreenEncoder(boolean sendFrameMeta, int bitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { @@ -54,6 +61,22 @@ public ScreenEncoder(boolean sendFrameMeta, int bitRate, int maxFps, List= 0) { + if (outputBufferId >= 0) { + try { + boolean configFrame = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; + boolean keyFrame = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0; + if (keyFrame && requestKeyFrameTime>0) { + long now = SystemClock.uptimeMillis(); + long ms = now - requestKeyFrameTime; + requestKeyFrameTime = 0; + Ln.d("Returning KEY FRAME after " + ms + "ms"); + } + else if (!keyFrame && !configFrame) { + Ln.d("Returning TWEEN FRAME"); + } + ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId); if (sendFrameMeta) { @@ -164,16 +234,17 @@ private boolean encode(MediaCodec codec, FileDescriptor fd) throws IOException { } IO.writeFully(fd, codecBuffer); - if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { + if (!configFrame) { // If this is not a config packet, then it contains a frame firstFrameSent = true; } - } - } finally { - if (outputBufferId >= 0) { + } finally { codec.releaseOutputBuffer(outputBufferId, false); } } + else { + // codec.flush(); + } } return !eof; @@ -249,11 +320,14 @@ private static MediaFormat createFormat(int bitRate, int maxFps, List 0) { // The key existed privately before Android 10: //