From 7125b15ef79661f3a15a8dbce07b558182669b05 Mon Sep 17 00:00:00 2001 From: Peter Abbondanzo Date: Wed, 2 Oct 2024 13:24:41 -0700 Subject: [PATCH] Add custom Fresco decoder for binary XML (#46711) Summary: Pull Request resolved: https://github.com/facebook/react-native/pull/46711 ## Summary Vector drawable decompression is a blocking operation and can sometimes take upwards of 20-30ms per image, especially if that vector drawable uses complex colors. Less complex vector drawables still take several milliseconds to decompress and are served from a cache on later load attempts. Here's a list of all the load times of vector drawable images in FBVR: {F1891592104} This diff aims to shift decompression to one of Fresco's decode threads, off the main thread, so we don't block while waiting for decompression operations to complete. This relies on adding a new custom decoder that reads the header of XML binary and converts encoded image requests to drawable objects that are yielded from the new `XmlDrawableFactory`. It's important to note that this change does not stop `ReactImageView` from loading XML-based drawables on the main thread, it merely offers a new mechanism for loading them. ## Changelog [Android][Added] - Add a new Fresco decoder for XML resource types Differential Revision: D63476283 --- .../ReactAndroid/api/ReactAndroid.api | 6 + .../react/modules/fresco/FrescoModule.kt | 22 ++- .../react/modules/fresco/XmlFormat.kt | 143 ++++++++++++++++++ 3 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 packages/react-native/ReactAndroid/src/main/java/com/facebook/react/modules/fresco/XmlFormat.kt diff --git a/packages/react-native/ReactAndroid/api/ReactAndroid.api b/packages/react-native/ReactAndroid/api/ReactAndroid.api index dbd95e722fb7f4..526aceb8d3d9e5 100644 --- a/packages/react-native/ReactAndroid/api/ReactAndroid.api +++ b/packages/react-native/ReactAndroid/api/ReactAndroid.api @@ -3443,6 +3443,12 @@ public final class com/facebook/react/modules/fresco/SystraceRequestListener : c public fun requiresExtraMap (Ljava/lang/String;)Z } +public final class com/facebook/react/modules/fresco/XmlFormat { + public static final field INSTANCE Lcom/facebook/react/modules/fresco/XmlFormat; + public final fun addDecodingCapability (Lcom/facebook/imagepipeline/decoder/ImageDecoderConfig$Builder;Landroid/content/Context;)Lcom/facebook/imagepipeline/decoder/ImageDecoderConfig$Builder; + public final fun getDrawableFactory ()Lcom/facebook/imagepipeline/drawable/DrawableFactory; +} + public final class com/facebook/react/modules/i18nmanager/I18nManagerModule : com/facebook/fbreact/specs/NativeI18nManagerSpec { public fun (Lcom/facebook/react/bridge/ReactApplicationContext;)V public fun allowRTL (Z)V diff --git a/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/modules/fresco/FrescoModule.kt b/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/modules/fresco/FrescoModule.kt index 43d08b45231a9a..e8ba89a6d0029a 100644 --- a/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/modules/fresco/FrescoModule.kt +++ b/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/modules/fresco/FrescoModule.kt @@ -8,17 +8,20 @@ package com.facebook.react.modules.fresco import com.facebook.common.logging.FLog +import com.facebook.drawee.backends.pipeline.DraweeConfig import com.facebook.drawee.backends.pipeline.Fresco import com.facebook.imagepipeline.backends.okhttp3.OkHttpImagePipelineConfigFactory.newBuilder import com.facebook.imagepipeline.core.DownsampleMode import com.facebook.imagepipeline.core.ImagePipeline import com.facebook.imagepipeline.core.ImagePipelineConfig +import com.facebook.imagepipeline.decoder.ImageDecoderConfig import com.facebook.imagepipeline.listener.RequestListener import com.facebook.react.bridge.LifecycleEventListener import com.facebook.react.bridge.ReactApplicationContext import com.facebook.react.bridge.ReactContext import com.facebook.react.bridge.ReactContextBaseJavaModule import com.facebook.react.common.ReactConstants +import com.facebook.react.internal.featureflags.ReactNativeFeatureFlags import com.facebook.react.module.annotations.ReactModule import com.facebook.react.modules.common.ModuleDataCleaner import com.facebook.react.modules.network.ForwardingCookieHandler @@ -75,10 +78,16 @@ constructor( val reactContext = reactApplicationContext reactContext.addLifecycleEventListener(this) if (!hasBeenInitialized()) { - if (config == null) { - config = getDefaultConfig(reactContext) + val pipelineConfig = config ?: getDefaultConfig(reactContext) + val draweeConfigBuilder = DraweeConfig.newBuilder() + if (ReactNativeFeatureFlags.loadVectorDrawablesOnImages()) { + draweeConfigBuilder.addCustomDrawableFactory(XmlFormat.getDrawableFactory()) } - Fresco.initialize(reactContext.applicationContext, config) + Fresco.initialize( + reactContext.applicationContext, + pipelineConfig, + draweeConfigBuilder.build(), + ) hasBeenInitialized = true } else if (config != null) { FLog.w( @@ -149,6 +158,12 @@ constructor( requestListeners.add(SystraceRequestListener()) val client = OkHttpClientProvider.createClient() + // Add support for XML drawable images + val decoderConfigBuilder = ImageDecoderConfig.Builder() + if (ReactNativeFeatureFlags.loadVectorDrawablesOnImages()) { + XmlFormat.addDecodingCapability(decoderConfigBuilder, context) + } + // make sure to forward cookies for any requests via the okHttpClient // so that image requests to endpoints that use cookies still work val container = OkHttpCompat.getCookieJarContainer(client) @@ -156,6 +171,7 @@ constructor( container.setCookieJar(JavaNetCookieJar(handler)) return newBuilder(context.applicationContext, client) .setNetworkFetcher(ReactOkHttpNetworkFetcher(client)) + .setImageDecoderConfig(decoderConfigBuilder.build()) .setDownsampleMode(DownsampleMode.AUTO) .setRequestListeners(requestListeners) } diff --git a/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/modules/fresco/XmlFormat.kt b/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/modules/fresco/XmlFormat.kt new file mode 100644 index 00000000000000..7b58dfa07abf50 --- /dev/null +++ b/packages/react-native/ReactAndroid/src/main/java/com/facebook/react/modules/fresco/XmlFormat.kt @@ -0,0 +1,143 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package com.facebook.react.modules.fresco + +import android.content.Context +import android.graphics.drawable.Drawable +import android.net.Uri +import com.facebook.common.logging.FLog +import com.facebook.imageformat.ImageFormat +import com.facebook.imageformat.ImageFormat.FormatChecker +import com.facebook.imageformat.ImageFormatCheckerUtils +import com.facebook.imagepipeline.common.ImageDecodeOptions +import com.facebook.imagepipeline.decoder.ImageDecoder +import com.facebook.imagepipeline.decoder.ImageDecoderConfig +import com.facebook.imagepipeline.drawable.DrawableFactory +import com.facebook.imagepipeline.image.CloseableImage +import com.facebook.imagepipeline.image.DefaultCloseableImage +import com.facebook.imagepipeline.image.EncodedImage +import com.facebook.imagepipeline.image.QualityInfo + +public object XmlFormat { + + public fun addDecodingCapability( + builder: ImageDecoderConfig.Builder, + context: Context, + ): ImageDecoderConfig.Builder { + return builder.addDecodingCapability( + XmlFormat.FORMAT, + XmlFormat.XmlFormatChecker(), + XmlFormat.XmlFormatDecoder(context), + ) + } + + public fun getDrawableFactory(): DrawableFactory { + return XmlDrawableFactory() + } + + private val FORMAT: ImageFormat = ImageFormat("XML", "xml") + private const val TAG: String = "XmlFormat" + /** + * These are the first 4 bytes of a binary XML file. We can only support binary XML files and not + * raw XML files because Android explicitly disallows raw XML files when inflating drawables. + * Binary XML files are created at build time by Android's AAPT. + * + * @see + * https://developer.android.com/reference/android/view/LayoutInflater#inflate(org.xmlpull.v1.XmlPullParser,%20android.view.ViewGroup) + */ + private val BINARY_XML_HEADER: ByteArray = + byteArrayOf( + 3.toByte(), + 0.toByte(), + 8.toByte(), + 0.toByte(), + ) + + private class XmlFormatChecker : FormatChecker { + override val headerSize: Int = BINARY_XML_HEADER.size + + override fun determineFormat(headerBytes: ByteArray, headerSize: Int): ImageFormat { + return when { + headerSize < BINARY_XML_HEADER.size -> ImageFormat.UNKNOWN + ImageFormatCheckerUtils.startsWithPattern(headerBytes, BINARY_XML_HEADER) -> FORMAT + else -> ImageFormat.UNKNOWN + } + } + } + + private class CloseableXmlImage(val name: String, val drawable: Drawable) : + DefaultCloseableImage() { + private var closed = false + + override fun getSizeInBytes(): Int { + return getWidth() * getHeight() * 4 // 4 bytes ARGB per pixel + } + + override fun close() { + closed = true + } + + override fun isClosed(): Boolean { + return closed + } + + override fun getWidth(): Int { + return drawable.intrinsicWidth.takeIf { it >= 0 } ?: 0 + } + + override fun getHeight(): Int { + return drawable.intrinsicHeight.takeIf { it >= 0 } ?: 0 + } + } + + private class XmlFormatDecoder(private val context: Context) : ImageDecoder { + override fun decode( + encodedImage: EncodedImage, + length: Int, + qualityInfo: QualityInfo, + options: ImageDecodeOptions + ): CloseableImage? { + return try { + val xmlResourceName = encodedImage.source ?: error("No source in encoded image") + // Use insecure URI parser since we do not care about the validity of the URI + val xmlResource = Uri.parse(xmlResourceName) + // Only support binary XML files from resources, not assets or raw files + val xmlResourceId = parseImageSourceResourceId(xmlResource) + // Use application context to avoid leaking the activity + val drawable = context.applicationContext.resources.getDrawable(xmlResourceId, null) + CloseableXmlImage(xmlResourceName, drawable) + } catch (error: Throwable) { + FLog.e(TAG, "Cannot decode xml ${error}", error) + null + } + } + + /** + * This parsing implementation is only designed to work with URI's that have been generated by + * the ResourceDrawableIdHelper that ImageSource uses. It will ignore package names and schemes + * in its quest to extract a basic integer resource ID. + * + * ResourceDrawableIdHelper generates URIs in the format of res:/[resourceId] + * + * @throws IllegalStateException if the resource ID cannot be parsed from the provided uri + */ + private fun parseImageSourceResourceId(xmlResource: Uri): Int { + return xmlResource.pathSegments.lastOrNull()?.toIntOrNull() ?: error("Invalid resource id") + } + } + + private class XmlDrawableFactory : DrawableFactory { + override fun supportsImageType(image: CloseableImage): Boolean { + return image is CloseableXmlImage + } + + override fun createDrawable(image: CloseableImage): Drawable? { + return (image as CloseableXmlImage).drawable + } + } +}