Record the content of external images during trace

gapii/cc/gles_extras.cpp

@@ -18,6 +18,7 @@
 #include "gapii/cc/gles_exports.h"
 #include "gapii/cc/gles_types.h"
 
+#include "gapis/api/gles/gles_pb/extras.pb.h"
 
 #define ANDROID_NATIVE_MAKE_CONSTANT(a,b,c,d) \
     (((unsigned)(a)<<24)|((unsigned)(b)<<16)|((unsigned)(c)<<8)|(unsigned)(d))
@@ -235,6 +236,189 @@ bool GlesSpy::getFramebufferAttachmentSize(CallObserver* observer, uint32_t* wid
     return false;
 }
 
+static bool ReadExternalPixels(GlesImports& imports, EGLImageKHR img, GLsizei width, GLsizei height, std::vector<uint8_t>* data) {
+    using namespace GLenum;
+
+    const char* vsSource =
+        "precision highp float;\n"
+        "attribute vec2 position;\n"
+        "varying vec2 texcoord;\n"
+        "void main() {\n"
+        "  gl_Position = vec4(position, 0.5, 1.0);\n"
+        "  texcoord = position * vec2(0.5) + vec2(0.5);\n"
+        "}\n";
+
+    const char* fsSource =
+        "#extension GL_OES_EGL_image_external : require\n"
+        "precision highp float;\n"
+        "uniform samplerExternalOES tex;\n"
+        "varying vec2 texcoord;\n"
+        "void main() {\n"
+        "  gl_FragColor = texture2D(tex, texcoord);\n"
+        "}\n";
+
+    GLint err;
+    auto prog = imports.glCreateProgram();
+
+    auto vs = imports.glCreateShader(GL_VERTEX_SHADER);
+    imports.glShaderSource(vs, 1, const_cast<char**>(&vsSource), nullptr);
+    imports.glCompileShader(vs);
+    imports.glAttachShader(prog, vs);
+
+    auto fs = imports.glCreateShader(GL_FRAGMENT_SHADER);
+    imports.glShaderSource(fs, 1, const_cast<char**>(&fsSource), nullptr);
+    imports.glCompileShader(fs);
+    imports.glAttachShader(prog, fs);
+
+    imports.glBindAttribLocation(prog, 0, "position");
+    imports.glLinkProgram(prog);
+
+    if ((err = imports.glGetError()) != 0) {
+        GAPID_ERROR("ReadExternalPixels: Failed to create program: 0x%X", err);
+        return false;
+    }
+
+    GLint linkStatus = 0;
+    imports.glGetProgramiv(prog, GL_LINK_STATUS, &linkStatus);
+    if (linkStatus == 0) {
+        char log[1024];
+        imports.glGetProgramInfoLog(prog, sizeof(log), nullptr, log);
+        GAPID_ERROR("ReadExternalPixels: Failed to compile program:\n%s", log);
+        return false;
+    }
+
+    GLuint srcTex = 0;
+    imports.glGenTextures(1, &srcTex);
+    imports.glBindTexture(GL_TEXTURE_EXTERNAL_OES, srcTex);
+    imports.glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, img);
+    imports.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+    imports.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+
+    GLuint dstTex = 0;
+    imports.glGenTextures(1, &dstTex);
+    imports.glBindTexture(GL_TEXTURE_2D, dstTex);
+    imports.glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr);
+
+    if ((err = imports.glGetError()) != 0) {
+        GAPID_ERROR("ReadExternalPixels: Failed to create texture: 0x%X", err);
+        return false;
+    }
+
+    GLuint fb = 0;
+    imports.glGenFramebuffers(1, &fb);
+    imports.glBindFramebuffer(GL_FRAMEBUFFER, fb);
+    imports.glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, dstTex, 0);
+
+    if ((err = imports.glGetError()) != 0) {
+        GAPID_ERROR("ReadExternalPixels: Failed to create framebuffer: 0x%X", err);
+        return false;
+    }
+    if ((err = imports.glCheckFramebufferStatus(GL_FRAMEBUFFER)) != GL_FRAMEBUFFER_COMPLETE) {
+        GAPID_ERROR("ReadExternalPixels: Framebuffer incomplete: 0x%X", err);
+        return false;
+    }
+
+    imports.glDisable(GL_CULL_FACE);
+    imports.glDisable(GL_DEPTH_TEST);
+    imports.glViewport(0, 0, width, height);
+    imports.glClearColor(0.0, 0.0, 0.0, 0.0);
+    imports.glClear(GLbitfield::GL_COLOR_BUFFER_BIT);
+    imports.glUseProgram(prog);
+    GLfloat vb[] = {
+        -1.0f, +1.0f,  // 2--4
+        -1.0f, -1.0f,  // |\ |
+        +1.0f, +1.0f,  // | \|
+        +1.0f, -1.0f,  // 1--3
+    };
+    imports.glEnableVertexAttribArray(0);
+    imports.glVertexAttribPointer(0, 2, GL_FLOAT, 0, 0, vb);
+    imports.glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+    if ((err = imports.glGetError()) != 0) {
+        GAPID_ERROR("ReadExternalPixels: Failed to draw quad: 0x%X", err);
+        return false;
+    }
+
+    data->resize(width * height * 4);
+    imports.glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, data->data());
+    if ((err = imports.glGetError()) != 0) {
+        GAPID_ERROR("ReadExternalPixels: Failed to read pixels: 0x%X", err);
+        return false;
+    }
+
+    return true;
+}
+
+void GlesSpy::GetEGLImageData(CallObserver* observer, EGLImageKHR img, GLsizei width, GLsizei height) {
+    using namespace EGLenum;
+
+    GAPID_DEBUG("Get EGLImage data: 0x%x %xx%x", img, width, height);
+
+    // Save old state.
+    auto display = mImports.eglGetCurrentDisplay();
+    auto draw = mImports.eglGetCurrentSurface(EGL_DRAW);
+    auto read = mImports.eglGetCurrentSurface(EGL_READ);
+    auto oldCtx = mImports.eglGetCurrentContext();
+
+    // Find an EGL config.
+    EGLConfig cfg;
+    EGLint cfgAttribs[] = { EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, EGL_NONE };
+    int one = 1;
+    if (mImports.eglChooseConfig(display, cfgAttribs, &cfg, 1, &one) == EGL_FALSE || one != 1) {
+        GAPID_ERROR("Failed to choose EGL config");
+        return;
+    }
+
+    // Create an EGL context.
+    EGLContext ctx;
+    EGLint ctxAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
+    if ((ctx = mImports.eglCreateContext(display, cfg, nullptr, ctxAttribs)) == nullptr) {
+        GAPID_ERROR("Failed to create EGL context");
+        return;
+    }
+
+    // Create an EGL surface.
+    EGLSurface surface;
+    EGLint surfaceAttribs[] = { EGL_WIDTH, 16, EGL_HEIGHT, 16, EGL_NONE };
+    if ((surface = mImports.eglCreatePbufferSurface(display, cfg, surfaceAttribs)) == nullptr) {
+        GAPID_ERROR("Failed to create EGL surface");
+        return;
+    }
+
+    // Bind the EGL context.
+    if (mImports.eglMakeCurrent(display, surface, surface, ctx) == EGL_FALSE) {
+        GAPID_ERROR("Failed to bind new EGL context");
+        return;
+    }
+
+    std::vector<uint8_t> data;
+    if (ReadExternalPixels(mImports, img, width, height, &data)) {
+        core::Id id = core::Id::Hash(data.data(), data.size());
+        if (getResources().count(id) == 0) {
+            capture::Resource resource;
+            resource.set_id(reinterpret_cast<const char*>(id.data), sizeof(id.data));
+            resource.set_data(data.data(), data.size());
+            getEncoder(kApiIndex)->object(&resource);
+            getResources().emplace(id);
+        }
+
+        auto extra = new gles_pb::EGLImageData();
+        extra->set_id(reinterpret_cast<const char*>(id.data), sizeof(id.data));
+        extra->set_size(data.size());
+        extra->set_width(width);
+        extra->set_height(height);
+        extra->set_format(GLenum::GL_RGBA);
+        extra->set_type(GLenum::GL_UNSIGNED_BYTE);
+        observer->encodeAndDelete(extra);
+    }
+
+    if (mImports.eglMakeCurrent(display, draw, read, oldCtx) == EGL_FALSE) {
+        GAPID_FATAL("Failed to restore old EGL context");
+    }
+
+    mImports.eglDestroySurface(display, surface);
+    mImports.eglDestroyContext(display, ctx);
+}
+
 bool GlesSpy::observeFramebuffer(CallObserver* observer, uint32_t* w, uint32_t* h, std::vector<uint8_t>* data) {
     if (!getFramebufferAttachmentSize(observer, w, h)) {
         return false; // Could not get the framebuffer size.

gapis/api/cmd_observations.go

@@ -114,7 +114,7 @@ func (o CmdObservation) String() string {
 	return fmt.Sprintf("{Range: %v, ID: %v}", o.Range, o.ID)
 }
 
-var _ ResourceReference = (*CmdObservation)(nil)
+var _ ResourceReference = CmdObservation{}
 
 // RemapResourceIDs remaps the serialized resource identifier with the
 // identifier used to store the resource in the database.

gapis/api/gles/api/extensions.api

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+@post_fence
+extern void GetEGLImageData(EGLImageKHR img, GLsizei width, GLsizei height)
+
 @if(Extension.GL_EXT_separate_shader_objects)
 @doc("https://www.khronos.org/registry/gles/extensions/EXT/EXT_separate_shader_objects.gles.txt", Extension.GL_EXT_separate_shader_objects)
 cmd void glActiveShaderProgramEXT(PipelineId pipeline, ProgramId program) {
@@ -641,6 +644,7 @@ cmd void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image) {
     tmpLevel.Layers[0] = eglImage.Image
     t.Levels[0] = tmpLevel
     t.EGLImage = eglImage
+    GetEGLImageData(eglImage.ID, eglImage.Image.Width, eglImage.Image.Height)
   }
 }
 

gapis/api/gles/compat.go

@@ -293,7 +293,7 @@ func compat(ctx context.Context, device *device.Instance) (transform.Transformer
 				cmd := *cmd
 				if cmd.Texture != 0 && !c.Objects.Shared.GeneratedNames.Textures[cmd.Texture] {
 					// glGenTextures() was not used to generate the texture. Legal in GLES 2.
-					tmp := s.AllocDataOrPanic(ctx, VertexArrayId(cmd.Texture))
+					tmp := s.AllocDataOrPanic(ctx, cmd.Texture)
 					out.MutateAndWrite(ctx, dID, cb.GlGenTextures(1, tmp.Ptr()).AddRead(tmp.Data()))
 				}
 
@@ -995,18 +995,23 @@ func compat(ctx context.Context, device *device.Instance) (transform.Transformer
 					}
 				case EGLenum_EGL_NATIVE_BUFFER_ANDROID:
 					{
-						texId := newTexture(id, cb, out)
+						texID := newTexture(id, cb, out)
 						t := newTweaker(out, dID, cb)
 						defer t.revert(ctx)
-						t.glBindTexture_2D(ctx, texId)
+						t.glBindTexture_2D(ctx, texID)
 						img := GetState(s).EGLImages[cmd.Result].Image
 						sizedFormat := img.SizedFormat // Might be RGB565 which is not supported on desktop
 						textureCompat.convertFormat(ctx, GLenum_GL_TEXTURE_2D, &sizedFormat, nil, nil, out, id, cmd)
 						out.MutateAndWrite(ctx, dID, cb.GlTexImage2D(GLenum_GL_TEXTURE_2D, 0, GLint(sizedFormat), img.Width, img.Height, 0, img.DataFormat, img.DataType, memory.Nullptr))
+						// Set the default filtering modes applicable to external images.
+						// This is important as the default (mipmap) mode would result in incomplete texture.
+						// TODO: Ensure that different contexts can set different modes at the same time.
+						out.MutateAndWrite(ctx, dID, cb.GlTexParameteri(GLenum_GL_TEXTURE_2D, GLenum_GL_TEXTURE_MIN_FILTER, GLint(GLenum_GL_LINEAR)))
+						out.MutateAndWrite(ctx, dID, cb.GlTexParameteri(GLenum_GL_TEXTURE_2D, GLenum_GL_TEXTURE_MAG_FILTER, GLint(GLenum_GL_LINEAR)))
 
 						out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
 							GetState(s).EGLImages[cmd.Result].TargetContext = c.Identifier
-							GetState(s).EGLImages[cmd.Result].TargetTexture = texId
+							GetState(s).EGLImages[cmd.Result].TargetTexture = texID
 							return nil
 						}))
 					}
@@ -1025,6 +1030,17 @@ func compat(ctx context.Context, device *device.Instance) (transform.Transformer
 				// Rebind the currently bound 2D texture.  This might seem like a no-op, however,
 				// the remapping layer will use the ID of the EGL image replacement texture now.
 				out.MutateAndWrite(ctx, dID, cb.GlBindTexture(GLenum_GL_TEXTURE_2D, c.Bound.TextureUnit.Binding2d.ID))
+
+				// Update the content if we made a snapshot.
+				if e := FindEGLImageData(cmd.Extras()); e != nil {
+					t := newTweaker(out, dID, cb)
+					defer t.revert(ctx)
+					t.setUnpackStorage(ctx, PixelStorageState{Alignment: 1}, 0)
+					ptr := s.AllocOrPanic(ctx, e.Size)
+					out.MutateAndWrite(ctx, dID, cb.GlTexSubImage2D(GLenum_GL_TEXTURE_2D, 0, 0, 0, e.Width, e.Height, e.Format, e.Type, ptr.Ptr()).AddRead(ptr.Range(), e.ID))
+					ptr.Free()
+				}
+
 				return
 			}
 

gapis/api/gles/externs.go

@@ -72,6 +72,23 @@ func (e externs) GetAndroidNativeBufferExtra(Voidᵖ) *AndroidNativeBufferExtra
 	return FindAndroidNativeBufferExtra(e.cmd.Extras())
 }
 
+func (e externs) GetEGLImageData(id EGLImageKHR, _ GLsizei, _ GLsizei) {
+	if d := FindEGLImageData(e.cmd.Extras()); d != nil {
+		if ei, ok := GetState(e.s).EGLImages[id]; ok {
+			if img := ei.Image; img != nil {
+				poolID, pool := e.s.Memory.New()
+				pool.Write(0, memory.Resource(d.ID, d.Size))
+				data := U8ˢ{pool: poolID, count: d.Size}
+				img.Width = GLsizei(d.Width)
+				img.Height = GLsizei(d.Height)
+				img.Data = data
+				img.DataFormat = d.Format
+				img.DataType = d.Type
+			}
+		}
+	}
+}
+
 func (e externs) calcIndexLimits(data U8ˢ, indexSize int) resolve.IndexRange {
 	id := data.ResourceID(e.ctx, e.s)
 	count := int(data.count) / int(indexSize)

gapis/api/gles/extras.go

@@ -18,6 +18,7 @@ import (
 	"context"
 
 	"github.com/google/gapid/core/data/deep"
+	"github.com/google/gapid/core/data/id"
 	"github.com/google/gapid/core/data/protoconv"
 	"github.com/google/gapid/gapis/api"
 	"github.com/google/gapid/gapis/api/gles/gles_pb"
@@ -30,6 +31,27 @@ type ErrorState struct {
 	InterceptorsGlError GLenum
 }
 
+// EGLImageData is an extra used to store snapshot of external image source.
+type EGLImageData struct {
+	ID     id.ID
+	Size   uint64
+	Width  GLsizei
+	Height GLsizei
+	Format GLenum
+	Type   GLenum
+}
+
+var _ api.ResourceReference = (*EGLImageData)(nil)
+
+// RemapResourceIDs remaps the serialized resource identifier with the
+// identifier used to store the resource in the database.
+func (e *EGLImageData) RemapResourceIDs(ids map[id.ID]id.ID) api.ResourceReference {
+	if id, found := ids[e.ID]; found {
+		e.ID = id
+	}
+	return e
+}
+
 func init() {
 	protoconv.Register(
 		func(ctx context.Context, o *ErrorState) (*gles_pb.ErrorState, error) {
@@ -44,6 +66,29 @@ func init() {
 			}, nil
 		},
 	)
+	protoconv.Register(
+		func(ctx context.Context, o *EGLImageData) (*gles_pb.EGLImageData, error) {
+			return &gles_pb.EGLImageData{
+				ID:     o.ID[:],
+				Size:   int32(o.Size),
+				Width:  int32(o.Width),
+				Height: int32(o.Height),
+				Format: int32(o.Format),
+				Type:   int32(o.Type),
+			}, nil
+		}, func(ctx context.Context, p *gles_pb.EGLImageData) (*EGLImageData, error) {
+			var id id.ID
+			copy(id[:], p.ID)
+			return &EGLImageData{
+				ID:     id,
+				Size:   uint64(p.Size),
+				Width:  GLsizei(p.Width),
+				Height: GLsizei(p.Height),
+				Format: GLenum(p.Format),
+				Type:   GLenum(p.Type),
+			}, nil
+		},
+	)
 }
 
 // FindProgramInfo searches for the ProgramInfo in the extras, returning the
@@ -72,6 +117,21 @@ func FindErrorState(extras *api.CmdExtras) *ErrorState {
 	return nil
 }
 
+// FindEGLImageData searches for the EGLImageData in the extras, returning the
+// EGLImageData if found, otherwise nil.
+func FindEGLImageData(extras *api.CmdExtras) *EGLImageData {
+	for _, e := range extras.All() {
+		if res, ok := e.(*EGLImageData); ok {
+			clone, err := deep.Clone(res)
+			if err != nil {
+				panic(err)
+			}
+			return clone.(*EGLImageData)
+		}
+	}
+	return nil
+}
+
 // FindStaticContextState searches for the StaticContextState in the extras,
 // returning the StaticContextState if found, otherwise nil.
 func FindStaticContextState(extras *api.CmdExtras) *StaticContextState {