From e9aafc809c30aac399281a7407b623d852c407a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Srbeck=C3=BD?= Date: Thu, 14 Sep 2017 15:12:20 +0100 Subject: [PATCH] Record the content of external images during trace Fixes #913 --- gapii/cc/gles_extras.cpp | 184 ++++++++++++++++++++++++++++ gapis/api/cmd_observations.go | 2 +- gapis/api/gles/api/extensions.api | 4 + gapis/api/gles/compat.go | 24 +++- gapis/api/gles/externs.go | 17 +++ gapis/api/gles/extras.go | 60 +++++++++ gapis/api/gles/gles_pb/extras.proto | 10 ++ gapis/api/gles/resources.go | 11 ++ 8 files changed, 307 insertions(+), 5 deletions(-) diff --git a/gapii/cc/gles_extras.cpp b/gapii/cc/gles_extras.cpp index 389865e2fd..1102e24d4e 100644 --- a/gapii/cc/gles_extras.cpp +++ b/gapii/cc/gles_extras.cpp @@ -18,6 +18,7 @@ #include "gapii/cc/gles_exports.h" #include "gapii/cc/gles_types.h" +#include "gapis/api/gles/gles_pb/extras.pb.h" #define ANDROID_NATIVE_MAKE_CONSTANT(a,b,c,d) \ (((unsigned)(a)<<24)|((unsigned)(b)<<16)|((unsigned)(c)<<8)|(unsigned)(d)) @@ -235,6 +236,189 @@ bool GlesSpy::getFramebufferAttachmentSize(CallObserver* observer, uint32_t* wid return false; } +static bool ReadExternalPixels(GlesImports& imports, EGLImageKHR img, GLsizei width, GLsizei height, std::vector* data) { + using namespace GLenum; + + const char* vsSource = + "precision highp float;\n" + "attribute vec2 position;\n" + "varying vec2 texcoord;\n" + "void main() {\n" + " gl_Position = vec4(position, 0.5, 1.0);\n" + " texcoord = position * vec2(0.5) + vec2(0.5);\n" + "}\n"; + + const char* fsSource = + "#extension GL_OES_EGL_image_external : require\n" + "precision highp float;\n" + "uniform samplerExternalOES tex;\n" + "varying vec2 texcoord;\n" + "void main() {\n" + " gl_FragColor = texture2D(tex, texcoord);\n" + "}\n"; + + GLint err; + auto prog = imports.glCreateProgram(); + + auto vs = imports.glCreateShader(GL_VERTEX_SHADER); + imports.glShaderSource(vs, 1, const_cast(&vsSource), nullptr); + imports.glCompileShader(vs); + imports.glAttachShader(prog, vs); + + auto fs = imports.glCreateShader(GL_FRAGMENT_SHADER); + imports.glShaderSource(fs, 1, const_cast(&fsSource), nullptr); + imports.glCompileShader(fs); + imports.glAttachShader(prog, fs); + + imports.glBindAttribLocation(prog, 0, "position"); + imports.glLinkProgram(prog); + + if ((err = imports.glGetError()) != 0) { + GAPID_ERROR("ReadExternalPixels: Failed to create program: 0x%X", err); + return false; + } + + GLint linkStatus = 0; + imports.glGetProgramiv(prog, GL_LINK_STATUS, &linkStatus); + if (linkStatus == 0) { + char log[1024]; + imports.glGetProgramInfoLog(prog, sizeof(log), nullptr, log); + GAPID_ERROR("ReadExternalPixels: Failed to compile program:\n%s", log); + return false; + } + + GLuint srcTex = 0; + imports.glGenTextures(1, &srcTex); + imports.glBindTexture(GL_TEXTURE_EXTERNAL_OES, srcTex); + imports.glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, img); + imports.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + imports.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + + GLuint dstTex = 0; + imports.glGenTextures(1, &dstTex); + imports.glBindTexture(GL_TEXTURE_2D, dstTex); + imports.glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + + if ((err = imports.glGetError()) != 0) { + GAPID_ERROR("ReadExternalPixels: Failed to create texture: 0x%X", err); + return false; + } + + GLuint fb = 0; + imports.glGenFramebuffers(1, &fb); + imports.glBindFramebuffer(GL_FRAMEBUFFER, fb); + imports.glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, dstTex, 0); + + if ((err = imports.glGetError()) != 0) { + GAPID_ERROR("ReadExternalPixels: Failed to create framebuffer: 0x%X", err); + return false; + } + if ((err = imports.glCheckFramebufferStatus(GL_FRAMEBUFFER)) != GL_FRAMEBUFFER_COMPLETE) { + GAPID_ERROR("ReadExternalPixels: Framebuffer incomplete: 0x%X", err); + return false; + } + + imports.glDisable(GL_CULL_FACE); + imports.glDisable(GL_DEPTH_TEST); + imports.glViewport(0, 0, width, height); + imports.glClearColor(0.0, 0.0, 0.0, 0.0); + imports.glClear(GLbitfield::GL_COLOR_BUFFER_BIT); + imports.glUseProgram(prog); + GLfloat vb[] = { + -1.0f, +1.0f, // 2--4 + -1.0f, -1.0f, // |\ | + +1.0f, +1.0f, // | \| + +1.0f, -1.0f, // 1--3 + }; + imports.glEnableVertexAttribArray(0); + imports.glVertexAttribPointer(0, 2, GL_FLOAT, 0, 0, vb); + imports.glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + if ((err = imports.glGetError()) != 0) { + GAPID_ERROR("ReadExternalPixels: Failed to draw quad: 0x%X", err); + return false; + } + + data->resize(width * height * 4); + imports.glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, data->data()); + if ((err = imports.glGetError()) != 0) { + GAPID_ERROR("ReadExternalPixels: Failed to read pixels: 0x%X", err); + return false; + } + + return true; +} + +void GlesSpy::GetEGLImageData(CallObserver* observer, EGLImageKHR img, GLsizei width, GLsizei height) { + using namespace EGLenum; + + GAPID_DEBUG("Get EGLImage data: 0x%x %xx%x", img, width, height); + + // Save old state. + auto display = mImports.eglGetCurrentDisplay(); + auto draw = mImports.eglGetCurrentSurface(EGL_DRAW); + auto read = mImports.eglGetCurrentSurface(EGL_READ); + auto oldCtx = mImports.eglGetCurrentContext(); + + // Find an EGL config. + EGLConfig cfg; + EGLint cfgAttribs[] = { EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, EGL_NONE }; + int one = 1; + if (mImports.eglChooseConfig(display, cfgAttribs, &cfg, 1, &one) == EGL_FALSE || one != 1) { + GAPID_ERROR("Failed to choose EGL config"); + return; + } + + // Create an EGL context. + EGLContext ctx; + EGLint ctxAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE }; + if ((ctx = mImports.eglCreateContext(display, cfg, nullptr, ctxAttribs)) == nullptr) { + GAPID_ERROR("Failed to create EGL context"); + return; + } + + // Create an EGL surface. + EGLSurface surface; + EGLint surfaceAttribs[] = { EGL_WIDTH, 16, EGL_HEIGHT, 16, EGL_NONE }; + if ((surface = mImports.eglCreatePbufferSurface(display, cfg, surfaceAttribs)) == nullptr) { + GAPID_ERROR("Failed to create EGL surface"); + return; + } + + // Bind the EGL context. + if (mImports.eglMakeCurrent(display, surface, surface, ctx) == EGL_FALSE) { + GAPID_ERROR("Failed to bind new EGL context"); + return; + } + + std::vector data; + if (ReadExternalPixels(mImports, img, width, height, &data)) { + core::Id id = core::Id::Hash(data.data(), data.size()); + if (getResources().count(id) == 0) { + capture::Resource resource; + resource.set_id(reinterpret_cast(id.data), sizeof(id.data)); + resource.set_data(data.data(), data.size()); + getEncoder(kApiIndex)->object(&resource); + getResources().emplace(id); + } + + auto extra = new gles_pb::EGLImageData(); + extra->set_id(reinterpret_cast(id.data), sizeof(id.data)); + extra->set_size(data.size()); + extra->set_width(width); + extra->set_height(height); + extra->set_format(GLenum::GL_RGBA); + extra->set_type(GLenum::GL_UNSIGNED_BYTE); + observer->encodeAndDelete(extra); + } + + if (mImports.eglMakeCurrent(display, draw, read, oldCtx) == EGL_FALSE) { + GAPID_FATAL("Failed to restore old EGL context"); + } + + mImports.eglDestroySurface(display, surface); + mImports.eglDestroyContext(display, ctx); +} + bool GlesSpy::observeFramebuffer(CallObserver* observer, uint32_t* w, uint32_t* h, std::vector* data) { if (!getFramebufferAttachmentSize(observer, w, h)) { return false; // Could not get the framebuffer size. diff --git a/gapis/api/cmd_observations.go b/gapis/api/cmd_observations.go index 3f1c5e3821..2313ec1d36 100644 --- a/gapis/api/cmd_observations.go +++ b/gapis/api/cmd_observations.go @@ -114,7 +114,7 @@ func (o CmdObservation) String() string { return fmt.Sprintf("{Range: %v, ID: %v}", o.Range, o.ID) } -var _ ResourceReference = (*CmdObservation)(nil) +var _ ResourceReference = CmdObservation{} // RemapResourceIDs remaps the serialized resource identifier with the // identifier used to store the resource in the database. diff --git a/gapis/api/gles/api/extensions.api b/gapis/api/gles/api/extensions.api index 2d0503cadc..b0dc79b767 100644 --- a/gapis/api/gles/api/extensions.api +++ b/gapis/api/gles/api/extensions.api @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +@post_fence +extern void GetEGLImageData(EGLImageKHR img, GLsizei width, GLsizei height) + @if(Extension.GL_EXT_separate_shader_objects) @doc("https://www.khronos.org/registry/gles/extensions/EXT/EXT_separate_shader_objects.gles.txt", Extension.GL_EXT_separate_shader_objects) cmd void glActiveShaderProgramEXT(PipelineId pipeline, ProgramId program) { @@ -641,6 +644,7 @@ cmd void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image) { tmpLevel.Layers[0] = eglImage.Image t.Levels[0] = tmpLevel t.EGLImage = eglImage + GetEGLImageData(eglImage.ID, eglImage.Image.Width, eglImage.Image.Height) } } diff --git a/gapis/api/gles/compat.go b/gapis/api/gles/compat.go index f113bb72b6..c5008fbc2d 100644 --- a/gapis/api/gles/compat.go +++ b/gapis/api/gles/compat.go @@ -293,7 +293,7 @@ func compat(ctx context.Context, device *device.Instance) (transform.Transformer cmd := *cmd if cmd.Texture != 0 && !c.Objects.Shared.GeneratedNames.Textures[cmd.Texture] { // glGenTextures() was not used to generate the texture. Legal in GLES 2. - tmp := s.AllocDataOrPanic(ctx, VertexArrayId(cmd.Texture)) + tmp := s.AllocDataOrPanic(ctx, cmd.Texture) out.MutateAndWrite(ctx, dID, cb.GlGenTextures(1, tmp.Ptr()).AddRead(tmp.Data())) } @@ -995,18 +995,23 @@ func compat(ctx context.Context, device *device.Instance) (transform.Transformer } case EGLenum_EGL_NATIVE_BUFFER_ANDROID: { - texId := newTexture(id, cb, out) + texID := newTexture(id, cb, out) t := newTweaker(out, dID, cb) defer t.revert(ctx) - t.glBindTexture_2D(ctx, texId) + t.glBindTexture_2D(ctx, texID) img := GetState(s).EGLImages[cmd.Result].Image sizedFormat := img.SizedFormat // Might be RGB565 which is not supported on desktop textureCompat.convertFormat(ctx, GLenum_GL_TEXTURE_2D, &sizedFormat, nil, nil, out, id, cmd) out.MutateAndWrite(ctx, dID, cb.GlTexImage2D(GLenum_GL_TEXTURE_2D, 0, GLint(sizedFormat), img.Width, img.Height, 0, img.DataFormat, img.DataType, memory.Nullptr)) + // Set the default filtering modes applicable to external images. + // This is important as the default (mipmap) mode would result in incomplete texture. + // TODO: Ensure that different contexts can set different modes at the same time. + out.MutateAndWrite(ctx, dID, cb.GlTexParameteri(GLenum_GL_TEXTURE_2D, GLenum_GL_TEXTURE_MIN_FILTER, GLint(GLenum_GL_LINEAR))) + out.MutateAndWrite(ctx, dID, cb.GlTexParameteri(GLenum_GL_TEXTURE_2D, GLenum_GL_TEXTURE_MAG_FILTER, GLint(GLenum_GL_LINEAR))) out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error { GetState(s).EGLImages[cmd.Result].TargetContext = c.Identifier - GetState(s).EGLImages[cmd.Result].TargetTexture = texId + GetState(s).EGLImages[cmd.Result].TargetTexture = texID return nil })) } @@ -1025,6 +1030,17 @@ func compat(ctx context.Context, device *device.Instance) (transform.Transformer // Rebind the currently bound 2D texture. This might seem like a no-op, however, // the remapping layer will use the ID of the EGL image replacement texture now. out.MutateAndWrite(ctx, dID, cb.GlBindTexture(GLenum_GL_TEXTURE_2D, c.Bound.TextureUnit.Binding2d.ID)) + + // Update the content if we made a snapshot. + if e := FindEGLImageData(cmd.Extras()); e != nil { + t := newTweaker(out, dID, cb) + defer t.revert(ctx) + t.setUnpackStorage(ctx, PixelStorageState{Alignment: 1}, 0) + ptr := s.AllocOrPanic(ctx, e.Size) + out.MutateAndWrite(ctx, dID, cb.GlTexSubImage2D(GLenum_GL_TEXTURE_2D, 0, 0, 0, e.Width, e.Height, e.Format, e.Type, ptr.Ptr()).AddRead(ptr.Range(), e.ID)) + ptr.Free() + } + return } diff --git a/gapis/api/gles/externs.go b/gapis/api/gles/externs.go index 3fbb5a2de8..33dc1ba381 100644 --- a/gapis/api/gles/externs.go +++ b/gapis/api/gles/externs.go @@ -72,6 +72,23 @@ func (e externs) GetAndroidNativeBufferExtra(Voidᵖ) *AndroidNativeBufferExtra return FindAndroidNativeBufferExtra(e.cmd.Extras()) } +func (e externs) GetEGLImageData(id EGLImageKHR, _ GLsizei, _ GLsizei) { + if d := FindEGLImageData(e.cmd.Extras()); d != nil { + if ei, ok := GetState(e.s).EGLImages[id]; ok { + if img := ei.Image; img != nil { + poolID, pool := e.s.Memory.New() + pool.Write(0, memory.Resource(d.ID, d.Size)) + data := U8ˢ{pool: poolID, count: d.Size} + img.Width = GLsizei(d.Width) + img.Height = GLsizei(d.Height) + img.Data = data + img.DataFormat = d.Format + img.DataType = d.Type + } + } + } +} + func (e externs) calcIndexLimits(data U8ˢ, indexSize int) resolve.IndexRange { id := data.ResourceID(e.ctx, e.s) count := int(data.count) / int(indexSize) diff --git a/gapis/api/gles/extras.go b/gapis/api/gles/extras.go index 34e55188ea..f5995162ed 100644 --- a/gapis/api/gles/extras.go +++ b/gapis/api/gles/extras.go @@ -18,6 +18,7 @@ import ( "context" "github.com/google/gapid/core/data/deep" + "github.com/google/gapid/core/data/id" "github.com/google/gapid/core/data/protoconv" "github.com/google/gapid/gapis/api" "github.com/google/gapid/gapis/api/gles/gles_pb" @@ -30,6 +31,27 @@ type ErrorState struct { InterceptorsGlError GLenum } +// EGLImageData is an extra used to store snapshot of external image source. +type EGLImageData struct { + ID id.ID + Size uint64 + Width GLsizei + Height GLsizei + Format GLenum + Type GLenum +} + +var _ api.ResourceReference = (*EGLImageData)(nil) + +// RemapResourceIDs remaps the serialized resource identifier with the +// identifier used to store the resource in the database. +func (e *EGLImageData) RemapResourceIDs(ids map[id.ID]id.ID) api.ResourceReference { + if id, found := ids[e.ID]; found { + e.ID = id + } + return e +} + func init() { protoconv.Register( func(ctx context.Context, o *ErrorState) (*gles_pb.ErrorState, error) { @@ -44,6 +66,29 @@ func init() { }, nil }, ) + protoconv.Register( + func(ctx context.Context, o *EGLImageData) (*gles_pb.EGLImageData, error) { + return &gles_pb.EGLImageData{ + ID: o.ID[:], + Size: int32(o.Size), + Width: int32(o.Width), + Height: int32(o.Height), + Format: int32(o.Format), + Type: int32(o.Type), + }, nil + }, func(ctx context.Context, p *gles_pb.EGLImageData) (*EGLImageData, error) { + var id id.ID + copy(id[:], p.ID) + return &EGLImageData{ + ID: id, + Size: uint64(p.Size), + Width: GLsizei(p.Width), + Height: GLsizei(p.Height), + Format: GLenum(p.Format), + Type: GLenum(p.Type), + }, nil + }, + ) } // FindProgramInfo searches for the ProgramInfo in the extras, returning the @@ -72,6 +117,21 @@ func FindErrorState(extras *api.CmdExtras) *ErrorState { return nil } +// FindEGLImageData searches for the EGLImageData in the extras, returning the +// EGLImageData if found, otherwise nil. +func FindEGLImageData(extras *api.CmdExtras) *EGLImageData { + for _, e := range extras.All() { + if res, ok := e.(*EGLImageData); ok { + clone, err := deep.Clone(res) + if err != nil { + panic(err) + } + return clone.(*EGLImageData) + } + } + return nil +} + // FindStaticContextState searches for the StaticContextState in the extras, // returning the StaticContextState if found, otherwise nil. func FindStaticContextState(extras *api.CmdExtras) *StaticContextState { diff --git a/gapis/api/gles/gles_pb/extras.proto b/gapis/api/gles/gles_pb/extras.proto index dfd92dc00f..0036b00472 100644 --- a/gapis/api/gles/gles_pb/extras.proto +++ b/gapis/api/gles/gles_pb/extras.proto @@ -22,3 +22,13 @@ message ErrorState { uint32 TraceDriversGlError = 1; uint32 InterceptorsGlError = 2; } + +message EGLImageData { + bytes ID = 1; // Resource ID of the image data + sint32 Size = 2; + sint32 Width = 3; + sint32 Height = 4; + sint32 Format = 5; // GLenum + sint32 Type = 6; // GLenum +} + diff --git a/gapis/api/gles/resources.go b/gapis/api/gles/resources.go index f0ae2f18f2..641862fdcb 100644 --- a/gapis/api/gles/resources.go +++ b/gapis/api/gles/resources.go @@ -77,6 +77,17 @@ func (t *Texture) ResourceData(ctx context.Context, s *api.GlobalState) (*api.Re panic(fmt.Errorf("Unhandled texture kind %v", t.Kind)) } + case GLenum_GL_TEXTURE_EXTERNAL_OES: + levels := make([]*image.Info, 1) + if ei := t.EGLImage; ei != nil && ei.Image != nil { + img, err := ei.Image.ImageInfo(ctx, s) + if err != nil { + return nil, err + } + levels[0] = img + } + return api.NewResourceData(api.NewTexture(&api.Texture2D{Levels: levels})), nil + case GLenum_GL_TEXTURE_1D_ARRAY: numLayers := t.LayerCount() layers := make([]*api.Texture1D, numLayers)