[SDL 0292] implement VirtualDisplayEncoder improvement for stable frame rate

android/sdl_android/src/androidTest/java/com/android/grafika/gles/OffscreenSurfaceTest.java

@@ -0,0 +1,105 @@
+package com.android.grafika.gles;
+
+import android.opengl.GLES20;
+import android.os.Environment;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import android.util.Log;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import static junit.framework.TestCase.assertTrue;
+
+@RunWith(AndroidJUnit4.class)
+public class OffscreenSurfaceTest {
+
+	private final String TAG = OffscreenSurfaceTest.class.getSimpleName();
+	private final int mWidth = 1280;
+	private final int mHeight = 720;
+	private final int mIterations = 100;
+
+	@Test
+	public void testReadPixels() {
+		EglCore eglCore = new EglCore(null, 0);
+		OffscreenSurface offscreenSurface = new OffscreenSurface(eglCore, mWidth, mHeight);
+		float time = runReadPixelsTest(offscreenSurface);
+		Log.d(TAG, "runReadPixelsTest returns " + time + " msec");
+	}
+
+	// HELPER test method
+	/**
+	 * Does a simple bit of rendering and then reads the pixels back.
+	 *
+	 * @return total time (msec order) spent on glReadPixels()
+	 */
+	private float runReadPixelsTest(OffscreenSurface eglSurface) {
+		long totalTime = 0;
+
+		eglSurface.makeCurrent();
+
+		ByteBuffer pixelBuf = ByteBuffer.allocateDirect(mWidth * mHeight * 4);
+		pixelBuf.order(ByteOrder.LITTLE_ENDIAN);
+
+		Log.d(TAG, "Running...");
+		float colorMult = 1.0f / mIterations;
+		for (int i = 0; i < mIterations; i++) {
+			if ((i % (mIterations / 8)) == 0) {
+				Log.d(TAG, "iteration " + i);
+			}
+
+			// Clear the screen to a solid color, then add a rectangle.  Change the color
+			// each time.
+			float r = i * colorMult;
+			float g = 1.0f - r;
+			float b = (r + g) / 2.0f;
+			GLES20.glClearColor(r, g, b, 1.0f);
+			GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+
+			GLES20.glEnable(GLES20.GL_SCISSOR_TEST);
+			GLES20.glScissor(mWidth / 4, mHeight / 4, mWidth / 2, mHeight / 2);
+			GLES20.glClearColor(b, g, r, 1.0f);
+			GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+			GLES20.glDisable(GLES20.GL_SCISSOR_TEST);
+
+			// Try to ensure that rendering has finished.
+			GLES20.glFinish();
+			GLES20.glReadPixels(0, 0, 1, 1,
+					GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf);
+
+			// Time individual extraction.  Ideally we'd be timing a bunch of these calls
+			// and measuring the aggregate time, but we want the isolated time, and if we
+			// just read the same buffer repeatedly we might get some sort of cache effect.
+			long startWhen = System.nanoTime();
+			GLES20.glReadPixels(0, 0, mWidth, mHeight,
+					GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf);
+			totalTime += System.nanoTime() - startWhen;
+		}
+		Log.d(TAG, "done");
+
+		// It's not the good idea to request external strage permission in unit test.
+		boolean requireStoragePermission = false;
+		if (requireStoragePermission) {
+			long startWhen = System.nanoTime();
+			File file = new File(Environment.getExternalStorageDirectory(),
+					"test.png");
+			try {
+				eglSurface.saveFrame(file);
+			} catch (IOException ioe) {
+				throw new RuntimeException(ioe);
+			}
+			Log.d(TAG, "Saved frame in " + ((System.nanoTime() - startWhen) / 1000000) + "ms");
+			assertTrue(file.exists());
+		} else {
+			// here' we can recognize Unit Test succeeded, but anyway checks to see totalTime and buffer capacity.
+			assertTrue(pixelBuf.capacity() > 0 && totalTime > 0);
+		}
+
+		return (float)totalTime / 1000000f;
+	}
+
+}

android/sdl_android/src/androidTest/java/com/smartdevicelink/managers/lifecycle/SystemCapabilityManagerTests.java

@@ -98,6 +98,7 @@ public void setUp() throws Exception {
         videoStreamingCapability.setMaxBitrate(TestValues.GENERAL_INT);
         videoStreamingCapability.setPreferredResolution(TestValues.GENERAL_IMAGERESOLUTION);
         videoStreamingCapability.setSupportedFormats(TestValues.GENERAL_VIDEOSTREAMINGFORMAT_LIST);
+        videoStreamingCapability.setPreferredFPS(TestValues.GENERAL_INTEGER);
         systemCapability.setCapabilityForType(SystemCapabilityType.VIDEO_STREAMING, videoStreamingCapability);
     }
 
@@ -214,6 +215,7 @@ public void testGetVSCapability() {
         vsCapability.setMaxBitrate(TestValues.GENERAL_INT);
         vsCapability.setPreferredResolution(TestValues.GENERAL_IMAGERESOLUTION);
         vsCapability.setSupportedFormats(TestValues.GENERAL_VIDEOSTREAMINGFORMAT_LIST);
+        vsCapability.setPreferredFPS(TestValues.GENERAL_INTEGER);
 
         SystemCapability cap = new SystemCapability();
         cap.setSystemCapabilityType(SystemCapabilityType.VIDEO_STREAMING);

android/sdl_android/src/androidTest/java/com/smartdevicelink/test/rpc/datatypes/VideoStreamingCapabilityTests.java

@@ -4,6 +4,7 @@
 import com.smartdevicelink.proxy.rpc.ImageResolution;
 import com.smartdevicelink.proxy.rpc.VideoStreamingCapability;
 import com.smartdevicelink.proxy.rpc.VideoStreamingFormat;
+import com.smartdevicelink.streaming.video.VideoStreamingParameters;
 import com.smartdevicelink.test.JsonUtils;
 import com.smartdevicelink.test.TestValues;
 import com.smartdevicelink.test.Validator;
@@ -33,6 +34,7 @@ public void setUp() {
         msg.setDiagonalScreenSize(TestValues.GENERAL_DOUBLE);
         msg.setPixelPerInch(TestValues.GENERAL_DOUBLE);
         msg.setScale(TestValues.GENERAL_DOUBLE);
+	    msg.setPreferredFPS(TestValues.GENERAL_INTEGER);
     }
 
     /**
@@ -47,6 +49,7 @@ public void testRpcValues() {
         Double diagonalScreenSize = msg.getDiagonalScreenSize();
         Double pixelPerInch = msg.getPixelPerInch();
         Double scale = msg.getScale();
+	    Integer preferredFPS = msg.getPreferredFPS();
 
         // Valid Tests
         assertEquals(TestValues.MATCH, (List<VideoStreamingFormat>) TestValues.GENERAL_VIDEOSTREAMINGFORMAT_LIST, format);
@@ -68,6 +71,7 @@ public void testRpcValues() {
         assertNull(TestValues.NULL, msg.getDiagonalScreenSize());
         assertNull(TestValues.NULL, msg.getPixelPerInch());
         assertNull(TestValues.NULL, msg.getScale());
+	    assertNull(TestValues.NULL, msg.getPreferredFPS());
     }
 
     public void testJson() {
@@ -81,6 +85,7 @@ public void testJson() {
             reference.put(VideoStreamingCapability.KEY_DIAGONAL_SCREEN_SIZE, TestValues.GENERAL_DOUBLE);
             reference.put(VideoStreamingCapability.KEY_PIXEL_PER_INCH, TestValues.GENERAL_DOUBLE);
             reference.put(VideoStreamingCapability.KEY_SCALE, TestValues.GENERAL_DOUBLE);
+	        reference.put(VideoStreamingCapability.KEY_PREFERRED_FPS, TestValues.GENERAL_INTEGER);
 
             JSONObject underTest = msg.serializeJSON();
             assertEquals(TestValues.MATCH, reference.length(), underTest.length());
@@ -89,7 +94,7 @@ public void testJson() {
             while (iterator.hasNext()) {
                 String key = (String) iterator.next();
 
-                if (key.equals(VideoStreamingCapability.KEY_MAX_BITRATE) || key.equals(VideoStreamingCapability.KEY_HAPTIC_SPATIAL_DATA_SUPPORTED)) {
+                if (key.equals(VideoStreamingCapability.KEY_MAX_BITRATE) || key.equals(VideoStreamingCapability.KEY_HAPTIC_SPATIAL_DATA_SUPPORTED) || key.equals(VideoStreamingCapability.KEY_PREFERRED_FPS)) {
                     assertTrue(TestValues.TRUE, JsonUtils.readIntegerFromJsonObject(reference, key) == JsonUtils.readIntegerFromJsonObject(underTest, key));
                 } else if (key.equals(VideoStreamingCapability.KEY_PREFERRED_RESOLUTION)) {
                     ImageResolution irReference = (ImageResolution) JsonUtils.readObjectFromJsonObject(reference, key);

android/sdl_android/src/main/java/com/android/grafika/gles/Drawable2d.java

@@ -0,0 +1,197 @@
+/*
+ * Copyright 2014 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.grafika.gles;
+
+import java.nio.FloatBuffer;
+
+/**
+ * Base class for stuff we like to draw.
+ */
+public class Drawable2d {
+    private static final int SIZEOF_FLOAT = 4;
+
+    /**
+     * Simple equilateral triangle (1.0 per side).  Centered on (0,0).
+     */
+    private static final float TRIANGLE_COORDS[] = {
+         0.0f,  0.577350269f,   // 0 top
+        -0.5f, -0.288675135f,   // 1 bottom left
+         0.5f, -0.288675135f    // 2 bottom right
+    };
+    private static final float TRIANGLE_TEX_COORDS[] = {
+        0.5f, 0.0f,     // 0 top center
+        0.0f, 1.0f,     // 1 bottom left
+        1.0f, 1.0f,     // 2 bottom right
+    };
+    private static final FloatBuffer TRIANGLE_BUF =
+            GlUtil.createFloatBuffer(TRIANGLE_COORDS);
+    private static final FloatBuffer TRIANGLE_TEX_BUF =
+            GlUtil.createFloatBuffer(TRIANGLE_TEX_COORDS);
+
+    /**
+     * Simple square, specified as a triangle strip.  The square is centered on (0,0) and has
+     * a size of 1x1.
+     * <p>
+     * Triangles are 0-1-2 and 2-1-3 (counter-clockwise winding).
+     */
+    private static final float RECTANGLE_COORDS[] = {
+        -0.5f, -0.5f,   // 0 bottom left
+         0.5f, -0.5f,   // 1 bottom right
+        -0.5f,  0.5f,   // 2 top left
+         0.5f,  0.5f,   // 3 top right
+    };
+    private static final float RECTANGLE_TEX_COORDS[] = {
+        0.0f, 1.0f,     // 0 bottom left
+        1.0f, 1.0f,     // 1 bottom right
+        0.0f, 0.0f,     // 2 top left
+        1.0f, 0.0f      // 3 top right
+    };
+    private static final FloatBuffer RECTANGLE_BUF =
+            GlUtil.createFloatBuffer(RECTANGLE_COORDS);
+    private static final FloatBuffer RECTANGLE_TEX_BUF =
+            GlUtil.createFloatBuffer(RECTANGLE_TEX_COORDS);
+
+    /**
+     * A "full" square, extending from -1 to +1 in both dimensions.  When the model/view/projection
+     * matrix is identity, this will exactly cover the viewport.
+     * <p>
+     * The texture coordinates are Y-inverted relative to RECTANGLE.  (This seems to work out
+     * right with external textures from SurfaceTexture.)
+     */
+    private static final float FULL_RECTANGLE_COORDS[] = {
+        -1.0f, -1.0f,   // 0 bottom left
+         1.0f, -1.0f,   // 1 bottom right
+        -1.0f,  1.0f,   // 2 top left
+         1.0f,  1.0f,   // 3 top right
+    };
+    private static final float FULL_RECTANGLE_TEX_COORDS[] = {
+        0.0f, 0.0f,     // 0 bottom left
+        1.0f, 0.0f,     // 1 bottom right
+        0.0f, 1.0f,     // 2 top left
+        1.0f, 1.0f      // 3 top right
+    };
+    private static final FloatBuffer FULL_RECTANGLE_BUF =
+            GlUtil.createFloatBuffer(FULL_RECTANGLE_COORDS);
+    private static final FloatBuffer FULL_RECTANGLE_TEX_BUF =
+            GlUtil.createFloatBuffer(FULL_RECTANGLE_TEX_COORDS);
+
+
+    private FloatBuffer mVertexArray;
+    private FloatBuffer mTexCoordArray;
+    private int mVertexCount;
+    private int mCoordsPerVertex;
+    private int mVertexStride;
+    private int mTexCoordStride;
+    private Prefab mPrefab;
+
+    /**
+     * Enum values for constructor.
+     */
+    public enum Prefab {
+        TRIANGLE, RECTANGLE, FULL_RECTANGLE
+    }
+
+    /**
+     * Prepares a drawable from a "pre-fabricated" shape definition.
+     * <p>
+     * Does no EGL/GL operations, so this can be done at any time.
+     */
+    public Drawable2d(Prefab shape) {
+        switch (shape) {
+            case TRIANGLE:
+                mVertexArray = TRIANGLE_BUF;
+                mTexCoordArray = TRIANGLE_TEX_BUF;
+                mCoordsPerVertex = 2;
+                mVertexStride = mCoordsPerVertex * SIZEOF_FLOAT;
+                mVertexCount = TRIANGLE_COORDS.length / mCoordsPerVertex;
+                break;
+            case RECTANGLE:
+                mVertexArray = RECTANGLE_BUF;
+                mTexCoordArray = RECTANGLE_TEX_BUF;
+                mCoordsPerVertex = 2;
+                mVertexStride = mCoordsPerVertex * SIZEOF_FLOAT;
+                mVertexCount = RECTANGLE_COORDS.length / mCoordsPerVertex;
+                break;
+            case FULL_RECTANGLE:
+                mVertexArray = FULL_RECTANGLE_BUF;
+                mTexCoordArray = FULL_RECTANGLE_TEX_BUF;
+                mCoordsPerVertex = 2;
+                mVertexStride = mCoordsPerVertex * SIZEOF_FLOAT;
+                mVertexCount = FULL_RECTANGLE_COORDS.length / mCoordsPerVertex;
+                break;
+            default:
+                throw new RuntimeException("Unknown shape " + shape);
+        }
+        mTexCoordStride = 2 * SIZEOF_FLOAT;
+        mPrefab = shape;
+    }
+
+    /**
+     * Returns the array of vertices.
+     * <p>
+     * To avoid allocations, this returns internal state.  The caller must not modify it.
+     */
+    public FloatBuffer getVertexArray() {
+        return mVertexArray;
+    }
+
+    /**
+     * Returns the array of texture coordinates.
+     * <p>
+     * To avoid allocations, this returns internal state.  The caller must not modify it.
+     */
+    public FloatBuffer getTexCoordArray() {
+        return mTexCoordArray;
+    }
+
+    /**
+     * Returns the number of vertices stored in the vertex array.
+     */
+    public int getVertexCount() {
+        return mVertexCount;
+    }
+
+    /**
+     * Returns the width, in bytes, of the data for each vertex.
+     */
+    public int getVertexStride() {
+        return mVertexStride;
+    }
+
+    /**
+     * Returns the width, in bytes, of the data for each texture coordinate.
+     */
+    public int getTexCoordStride() {
+        return mTexCoordStride;
+    }
+
+    /**
+     * Returns the number of position coordinates per vertex.  This will be 2 or 3.
+     */
+    public int getCoordsPerVertex() {
+        return mCoordsPerVertex;
+    }
+
+    @Override
+    public String toString() {
+        if (mPrefab != null) {
+            return "[Drawable2d: " + mPrefab + "]";
+        } else {
+            return "[Drawable2d: ...]";
+        }
+    }
+}